2024-11-19 07:32:24,193 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-19 07:32:24,210 main DEBUG Took 0.015120 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-19 07:32:24,211 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-19 07:32:24,211 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-19 07:32:24,213 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-19 07:32:24,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,225 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-19 07:32:24,251 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,254 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,255 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,257 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,258 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,259 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,260 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,262 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,264 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,265 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-19 07:32:24,266 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,267 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-19 07:32:24,269 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-19 07:32:24,270 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-19 07:32:24,273 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-19 07:32:24,274 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-19 07:32:24,276 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-19 07:32:24,277 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-19 07:32:24,287 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-19 07:32:24,290 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-19 07:32:24,292 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-19 07:32:24,292 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-19 07:32:24,293 main DEBUG createAppenders(={Console})
2024-11-19 07:32:24,294 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-19 07:32:24,294 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-19 07:32:24,294 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-19 07:32:24,295 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-19 07:32:24,296 main DEBUG OutputStream closed
2024-11-19 07:32:24,296 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-19 07:32:24,296 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-19 07:32:24,297 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-19 07:32:24,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-19 07:32:24,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-19 07:32:24,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-19 07:32:24,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-19 07:32:24,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-19 07:32:24,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-19 07:32:24,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-19 07:32:24,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-19 07:32:24,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-19 07:32:24,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-19 07:32:24,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-19 07:32:24,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-19 07:32:24,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-19 07:32:24,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-19 07:32:24,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-19 07:32:24,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-19 07:32:24,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-19 07:32:24,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-19 07:32:24,442 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-19 07:32:24,443 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-19 07:32:24,443 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-19 07:32:24,445 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-19T07:32:24,875 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01
2024-11-19 07:32:24,878 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-19 07:32:24,879 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-19T07:32:24,890 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-19T07:32:24,959 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=13082
2024-11-19T07:32:24,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:32:24,986 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453, deleteOnExit=true
2024-11-19T07:32:24,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:32:24,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/test.cache.data in system properties and HBase conf
2024-11-19T07:32:24,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:32:24,993 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:32:24,994 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:32:24,995 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:32:24,995 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:32:25,126 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-19T07:32:25,250 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:32:25,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:32:25,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:32:25,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:32:25,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:32:25,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:32:25,259 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:32:25,260 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:32:25,261 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:32:25,262 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:32:25,262 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:32:25,263 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:32:25,264 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:32:25,265 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:32:25,266 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:32:25,929 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:32:26,713 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-19T07:32:26,807 INFO  [Time-limited test {}] log.Log(170): Logging initialized @3495ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-19T07:32:26,897 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:32:26,976 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:32:27,006 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:32:27,007 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:32:27,009 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:32:27,025 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:32:27,029 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:32:27,030 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:32:27,230 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/java.io.tmpdir/jetty-localhost-45563-hadoop-hdfs-3_4_1-tests_jar-_-any-3790089251846632495/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:32:27,236 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:45563}
2024-11-19T07:32:27,236 INFO  [Time-limited test {}] server.Server(415): Started @3925ms
2024-11-19T07:32:27,266 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:32:27,789 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:32:27,796 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:32:27,797 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:32:27,797 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:32:27,798 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:32:27,798 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:32:27,799 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:32:27,902 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/java.io.tmpdir/jetty-localhost-38245-hadoop-hdfs-3_4_1-tests_jar-_-any-358447321982370003/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:32:27,903 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:38245}
2024-11-19T07:32:27,903 INFO  [Time-limited test {}] server.Server(415): Started @4592ms
2024-11-19T07:32:27,953 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:32:28,055 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:32:28,062 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:32:28,067 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:32:28,068 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:32:28,068 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:32:28,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:32:28,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:32:28,184 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/java.io.tmpdir/jetty-localhost-34053-hadoop-hdfs-3_4_1-tests_jar-_-any-9311062972220463799/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:32:28,185 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:34053}
2024-11-19T07:32:28,185 INFO  [Time-limited test {}] server.Server(415): Started @4874ms
2024-11-19T07:32:28,188 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:32:29,675 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data3/current/BP-2111989087-172.17.0.2-1732001546043/current, will proceed with Du for space computation calculation, 
2024-11-19T07:32:29,675 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data2/current/BP-2111989087-172.17.0.2-1732001546043/current, will proceed with Du for space computation calculation, 
2024-11-19T07:32:29,675 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data4/current/BP-2111989087-172.17.0.2-1732001546043/current, will proceed with Du for space computation calculation, 
2024-11-19T07:32:29,675 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data1/current/BP-2111989087-172.17.0.2-1732001546043/current, will proceed with Du for space computation calculation, 
2024-11-19T07:32:29,705 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:32:29,705 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:32:29,753 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x31db1e1a3f965eb5 with lease ID 0x43dee719f02dcef5: Processing first storage report for DS-aedb410d-3b10-4698-a67d-0328ef4006a9 from datanode DatanodeRegistration(127.0.0.1:34237, datanodeUuid=538eaccd-ca9b-4a2a-a614-9beed8fe8f44, infoPort=36067, infoSecurePort=0, ipcPort=37545, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043)
2024-11-19T07:32:29,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x31db1e1a3f965eb5 with lease ID 0x43dee719f02dcef5: from storage DS-aedb410d-3b10-4698-a67d-0328ef4006a9 node DatanodeRegistration(127.0.0.1:34237, datanodeUuid=538eaccd-ca9b-4a2a-a614-9beed8fe8f44, infoPort=36067, infoSecurePort=0, ipcPort=37545, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T07:32:29,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa92f0de71e1e4f9a with lease ID 0x43dee719f02dcef4: Processing first storage report for DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942 from datanode DatanodeRegistration(127.0.0.1:34019, datanodeUuid=596598a2-c950-4c83-8fa3-964c717d5c3d, infoPort=36895, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043)
2024-11-19T07:32:29,755 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa92f0de71e1e4f9a with lease ID 0x43dee719f02dcef4: from storage DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942 node DatanodeRegistration(127.0.0.1:34019, datanodeUuid=596598a2-c950-4c83-8fa3-964c717d5c3d, infoPort=36895, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:32:29,755 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x31db1e1a3f965eb5 with lease ID 0x43dee719f02dcef5: Processing first storage report for DS-1873e8e3-3a82-4ce9-857e-8c9613199a82 from datanode DatanodeRegistration(127.0.0.1:34237, datanodeUuid=538eaccd-ca9b-4a2a-a614-9beed8fe8f44, infoPort=36067, infoSecurePort=0, ipcPort=37545, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043)
2024-11-19T07:32:29,755 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x31db1e1a3f965eb5 with lease ID 0x43dee719f02dcef5: from storage DS-1873e8e3-3a82-4ce9-857e-8c9613199a82 node DatanodeRegistration(127.0.0.1:34237, datanodeUuid=538eaccd-ca9b-4a2a-a614-9beed8fe8f44, infoPort=36067, infoSecurePort=0, ipcPort=37545, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T07:32:29,755 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa92f0de71e1e4f9a with lease ID 0x43dee719f02dcef4: Processing first storage report for DS-7eb29413-8a35-4852-872a-20f56b26c448 from datanode DatanodeRegistration(127.0.0.1:34019, datanodeUuid=596598a2-c950-4c83-8fa3-964c717d5c3d, infoPort=36895, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043)
2024-11-19T07:32:29,755 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa92f0de71e1e4f9a with lease ID 0x43dee719f02dcef4: from storage DS-7eb29413-8a35-4852-872a-20f56b26c448 node DatanodeRegistration(127.0.0.1:34019, datanodeUuid=596598a2-c950-4c83-8fa3-964c717d5c3d, infoPort=36895, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=725767103;c=1732001546043), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:32:29,844 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01
2024-11-19T07:32:29,910 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/zookeeper_0, clientPort=53123, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:32:29,919 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53123
2024-11-19T07:32:29,928 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:29,930 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:30,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:32:30,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:32:30,536 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b with version=8
2024-11-19T07:32:30,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:32:30,611 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-19T07:32:30,841 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:32:30,849 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:30,850 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:30,854 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:32:30,854 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:30,854 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:32:31,005 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:32:31,082 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-19T07:32:31,094 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-19T07:32:31,097 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:32:31,123 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 98036 (auto-detected)
2024-11-19T07:32:31,124 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-19T07:32:31,145 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44713
2024-11-19T07:32:31,169 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44713 connecting to ZooKeeper ensemble=127.0.0.1:53123
2024-11-19T07:32:31,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447130x0, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:32:31,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44713-0x101520249ee0000 connected
2024-11-19T07:32:31,457 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:31,461 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:31,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:32:31,477 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b, hbase.cluster.distributed=false
2024-11-19T07:32:31,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:32:31,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44713
2024-11-19T07:32:31,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44713
2024-11-19T07:32:31,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44713
2024-11-19T07:32:31,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44713
2024-11-19T07:32:31,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44713
2024-11-19T07:32:31,658 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:32:31,660 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:31,661 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:31,661 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:32:31,661 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:32:31,662 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:32:31,666 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:32:31,669 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:32:31,671 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34769
2024-11-19T07:32:31,674 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34769 connecting to ZooKeeper ensemble=127.0.0.1:53123
2024-11-19T07:32:31,675 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:31,683 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:31,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347690x0, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:32:31,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:32:31,704 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34769-0x101520249ee0001 connected
2024-11-19T07:32:31,710 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:32:31,719 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:32:31,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:32:31,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:32:31,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34769
2024-11-19T07:32:31,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34769
2024-11-19T07:32:31,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34769
2024-11-19T07:32:31,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34769
2024-11-19T07:32:31,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34769
2024-11-19T07:32:31,757 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:44713
2024-11-19T07:32:31,758 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,44713,1732001550692
2024-11-19T07:32:31,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:32:31,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:32:31,780 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,44713,1732001550692
2024-11-19T07:32:31,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:32:31,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:31,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:31,816 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:32:31,819 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,44713,1732001550692 from backup master directory
2024-11-19T07:32:31,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:32:31,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,44713,1732001550692
2024-11-19T07:32:31,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:32:31,834 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:32:31,835 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,44713,1732001550692
2024-11-19T07:32:31,838 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-19T07:32:31,841 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-19T07:32:31,915 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase.id] with ID: cc282e08-583d-4a12-a458-13078b940c86
2024-11-19T07:32:31,915 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/.tmp/hbase.id
2024-11-19T07:32:31,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:32:31,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:32:31,931 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/.tmp/hbase.id]:[hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase.id]
2024-11-19T07:32:31,977 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:31,983 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:32:32,005 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms.
2024-11-19T07:32:32,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:32,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:32,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:32:32,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:32:32,470 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:32:32,472 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:32:32,477 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:32:32,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:32:32,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:32:32,529 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store
2024-11-19T07:32:32,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:32:32,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:32:32,555 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-19T07:32:32,559 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:32,561 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:32:32,561 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:32:32,561 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:32:32,563 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:32:32,563 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:32:32,563 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:32:32,564 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001552560Disabling compacts and flushes for region at 1732001552560Disabling writes for close at 1732001552563 (+3 ms)Writing region close event to WAL at 1732001552563Closed at 1732001552563
2024-11-19T07:32:32,567 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/.initializing
2024-11-19T07:32:32,567 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/WALs/db0afee3eab9,44713,1732001550692
2024-11-19T07:32:32,592 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C44713%2C1732001550692, suffix=, logDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/WALs/db0afee3eab9,44713,1732001550692, archiveDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/oldWALs, maxLogs=10
2024-11-19T07:32:32,602 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C44713%2C1732001550692.1732001552597
2024-11-19T07:32:32,624 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/WALs/db0afee3eab9,44713,1732001550692/db0afee3eab9%2C44713%2C1732001550692.1732001552597
2024-11-19T07:32:32,638 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:32:32,640 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:32:32,641 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:32,644 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,645 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,685 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:32:32,716 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:32:32,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:32,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:32,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:32:32,728 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:32:32,728 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:32,730 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:32:32,731 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:32:32,735 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:32:32,735 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:32,737 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:32:32,737 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:32:32,742 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:32:32,742 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:32,744 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:32:32,744 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,749 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,751 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,758 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,759 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,764 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:32:32,774 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:32:32,792 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:32:32,794 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739613, jitterRate=-0.05953344702720642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:32:32,805 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001552662Initializing all the Stores at 1732001552664 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001552665 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001552666 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001552667 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001552667Cleaning up temporary data from old regions at 1732001552759 (+92 ms)Region opened successfully at 1732001552805 (+46 ms)
2024-11-19T07:32:32,807 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:32:32,856 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f33e2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:32:32,898 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:32:32,916 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:32:32,917 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:32:32,921 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:32:32,923 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-19T07:32:32,930 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-19T07:32:32,930 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:32:32,960 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:32:32,974 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:32:33,023 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:32:33,025 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:32:33,028 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:32:33,039 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:32:33,042 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:32:33,045 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:32:33,054 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:32:33,056 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:32:33,065 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:32:33,090 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:32:33,103 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:32:33,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:32:33,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:32:33,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,121 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,44713,1732001550692, sessionid=0x101520249ee0000, setting cluster-up flag (Was=false)
2024-11-19T07:32:33,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,181 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:32:33,183 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,44713,1732001550692
2024-11-19T07:32:33,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,240 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:32:33,242 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,44713,1732001550692
2024-11-19T07:32:33,248 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:32:33,320 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:32:33,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:32:33,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:32:33,343 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,44713,1732001550692
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:32:33,347 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(746): ClusterId : cc282e08-583d-4a12-a458-13078b940c86
2024-11-19T07:32:33,349 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:32:33,351 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:32:33,352 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:32:33,352 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:32:33,352 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:32:33,352 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:32:33,353 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,353 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:32:33,353 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,354 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001583354
2024-11-19T07:32:33,356 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:32:33,357 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:32:33,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:32:33,359 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:32:33,361 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:32:33,361 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:32:33,362 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:32:33,362 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:32:33,362 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:32:33,362 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:32:33,363 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,365 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:33,365 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:32:33,366 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:32:33,367 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:32:33,368 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:32:33,369 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:32:33,370 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:32:33,372 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001553371,5,FailOnTimeoutGroup]
2024-11-19T07:32:33,373 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:32:33,374 DEBUG [RS:0;db0afee3eab9:34769 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25644965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:32:33,376 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001553373,5,FailOnTimeoutGroup]
2024-11-19T07:32:33,376 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,377 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:32:33,378 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,378 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:32:33,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:32:33,387 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:32:33,387 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b
2024-11-19T07:32:33,391 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:34769
2024-11-19T07:32:33,394 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:32:33,394 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:32:33,395 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:32:33,398 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,44713,1732001550692 with port=34769, startcode=1732001551610
2024-11-19T07:32:33,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:32:33,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:32:33,412 DEBUG [RS:0;db0afee3eab9:34769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:32:33,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:33,416 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:32:33,419 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:32:33,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:33,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:33,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:32:33,424 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:32:33,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:33,426 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:33,426 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:32:33,429 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:32:33,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:33,431 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:33,431 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:32:33,438 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:32:33,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:33,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:33,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:32:33,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740
2024-11-19T07:32:33,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740
2024-11-19T07:32:33,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:32:33,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:32:33,450 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:32:33,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:32:33,460 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:32:33,461 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835468, jitterRate=0.062353432178497314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:32:33,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001553414Initializing all the Stores at 1732001553416 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001553416Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001553416Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001553416Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001553416Cleaning up temporary data from old regions at 1732001553449 (+33 ms)Region opened successfully at 1732001553466 (+17 ms)
2024-11-19T07:32:33,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:32:33,467 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:32:33,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:32:33,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:32:33,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:32:33,469 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:32:33,469 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001553467Disabling compacts and flushes for region at 1732001553467Disabling writes for close at 1732001553467Writing region close event to WAL at 1732001553468 (+1 ms)Closed at 1732001553469 (+1 ms)
2024-11-19T07:32:33,473 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:32:33,473 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:32:33,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:32:33,500 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45871, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:32:33,505 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:32:33,509 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:32:33,517 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44713 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,519 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44713 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,534 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b
2024-11-19T07:32:33,534 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45255
2024-11-19T07:32:33,534 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:32:33,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:32:33,550 DEBUG [RS:0;db0afee3eab9:34769 {}] zookeeper.ZKUtil(111): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,550 WARN  [RS:0;db0afee3eab9:34769 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:32:33,550 INFO  [RS:0;db0afee3eab9:34769 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:32:33,551 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,552 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,34769,1732001551610]
2024-11-19T07:32:33,574 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:32:33,589 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:32:33,593 INFO  [RS:0;db0afee3eab9:34769 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:32:33,593 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,594 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:32:33,599 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:32:33,600 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,600 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,601 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:32:33,602 DEBUG [RS:0;db0afee3eab9:34769 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:32:33,603 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,603 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,603 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,603 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,603 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,604 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,34769,1732001551610-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:32:33,619 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:32:33,621 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,34769,1732001551610-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,621 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,621 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.Replication(171): db0afee3eab9,34769,1732001551610 started
2024-11-19T07:32:33,641 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:33,642 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,34769,1732001551610, RpcServer on db0afee3eab9/172.17.0.2:34769, sessionid=0x101520249ee0001
2024-11-19T07:32:33,643 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:32:33,643 DEBUG [RS:0;db0afee3eab9:34769 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,643 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,34769,1732001551610'
2024-11-19T07:32:33,643 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:32:33,645 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:32:33,645 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:32:33,646 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:32:33,646 DEBUG [RS:0;db0afee3eab9:34769 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,646 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,34769,1732001551610'
2024-11-19T07:32:33,646 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:32:33,647 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:32:33,647 DEBUG [RS:0;db0afee3eab9:34769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:32:33,648 INFO  [RS:0;db0afee3eab9:34769 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:32:33,648 INFO  [RS:0;db0afee3eab9:34769 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:32:33,667 WARN  [db0afee3eab9:44713 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:32:33,759 INFO  [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C34769%2C1732001551610, suffix=, logDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610, archiveDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs, maxLogs=32
2024-11-19T07:32:33,762 INFO  [RS:0;db0afee3eab9:34769 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001553761
2024-11-19T07:32:33,775 INFO  [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001553761
2024-11-19T07:32:33,779 DEBUG [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:32:33,919 DEBUG [db0afee3eab9:44713 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:32:33,932 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:33,937 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,34769,1732001551610, state=OPENING
2024-11-19T07:32:33,949 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:32:33,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:32:33,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:32:33,960 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:32:33,962 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:32:33,965 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,34769,1732001551610}]
2024-11-19T07:32:34,147 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:32:34,150 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45853, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:32:34,162 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:32:34,163 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:32:34,166 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C34769%2C1732001551610.meta, suffix=.meta, logDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610, archiveDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs, maxLogs=32
2024-11-19T07:32:34,169 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.meta.1732001554168.meta
2024-11-19T07:32:34,176 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.meta.1732001554168.meta
2024-11-19T07:32:34,178 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:36895:36895)]
2024-11-19T07:32:34,180 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:32:34,181 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:32:34,184 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:32:34,188 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:32:34,192 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:32:34,193 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:34,193 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:32:34,193 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:32:34,196 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:32:34,199 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:32:34,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:34,200 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:34,201 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:32:34,203 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:32:34,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:34,204 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:34,204 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:32:34,206 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:32:34,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:34,207 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:34,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:32:34,209 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:32:34,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:34,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:32:34,211 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:32:34,212 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740
2024-11-19T07:32:34,215 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740
2024-11-19T07:32:34,218 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:32:34,218 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:32:34,220 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:32:34,223 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:32:34,225 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817068, jitterRate=0.03895579278469086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:32:34,225 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:32:34,226 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001554194Writing region info on filesystem at 1732001554194Initializing all the Stores at 1732001554196 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001554196Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001554196Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001554196Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001554196Cleaning up temporary data from old regions at 1732001554219 (+23 ms)Running coprocessor post-open hooks at 1732001554225 (+6 ms)Region opened successfully at 1732001554226 (+1 ms)
2024-11-19T07:32:34,233 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001554140
2024-11-19T07:32:34,247 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:32:34,248 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:32:34,249 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:34,251 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,34769,1732001551610, state=OPEN
2024-11-19T07:32:34,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:32:34,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:32:34,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:32:34,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:32:34,343 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:34,351 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:32:34,351 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,34769,1732001551610 in 378 msec
2024-11-19T07:32:34,360 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:32:34,360 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 872 msec
2024-11-19T07:32:34,363 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:32:34,364 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:32:34,388 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:32:34,390 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,34769,1732001551610, seqNum=-1]
2024-11-19T07:32:34,410 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:32:34,412 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42787, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:32:34,433 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1540 sec
2024-11-19T07:32:34,433 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001554433, completionTime=-1
2024-11-19T07:32:34,436 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:32:34,437 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:32:34,464 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:32:34,464 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001614464
2024-11-19T07:32:34,464 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001674464
2024-11-19T07:32:34,464 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec
2024-11-19T07:32:34,467 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,468 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,468 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,469 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:44713, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,470 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,471 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,477 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:32:34,502 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.667sec
2024-11-19T07:32:34,503 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:32:34,505 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:32:34,506 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:32:34,506 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:32:34,506 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:32:34,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:32:34,508 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:32:34,517 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:32:34,518 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:32:34,518 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,44713,1732001550692-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:32:34,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:32:34,561 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-19T07:32:34,561 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-19T07:32:34,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,44713,-1 for getting cluster id
2024-11-19T07:32:34,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:32:34,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cc282e08-583d-4a12-a458-13078b940c86'
2024-11-19T07:32:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:32:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cc282e08-583d-4a12-a458-13078b940c86"

2024-11-19T07:32:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f22587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:32:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,44713,-1]
2024-11-19T07:32:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:32:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:32:34,589 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:32:34,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:32:34,593 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:32:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,34769,1732001551610, seqNum=-1]
2024-11-19T07:32:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:32:34,605 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:32:34,625 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,44713,1732001550692
2024-11-19T07:32:34,625 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:32:34,633 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:32:34,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T07:32:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,44713,1732001550692
2024-11-19T07:32:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@290bf544
2024-11-19T07:32:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T07:32:34,666 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T07:32:34,668 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T07:32:34,669 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T07:32:34,673 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:32:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-19T07:32:34,687 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T07:32:34,697 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:34,700 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T07:32:34,708 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-19T07:32:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:32:34,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741835_1011 (size=389)
2024-11-19T07:32:34,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741835_1011 (size=389)
2024-11-19T07:32:34,750 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33cb0d40cd2d185d3207eca50152e3cc, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b
2024-11-19T07:32:34,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741836_1012 (size=72)
2024-11-19T07:32:34,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741836_1012 (size=72)
2024-11-19T07:32:34,767 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:34,767 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 33cb0d40cd2d185d3207eca50152e3cc, disabling compactions & flushes
2024-11-19T07:32:34,767 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:34,767 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:34,767 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc. after waiting 0 ms
2024-11-19T07:32:34,767 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:34,768 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:34,768 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33cb0d40cd2d185d3207eca50152e3cc:
Waiting for close lock at 1732001554767Disabling compacts and flushes for region at 1732001554767Disabling writes for close at 1732001554767Writing region close event to WAL at 1732001554768 (+1 ms)Closed at 1732001554768
2024-11-19T07:32:34,771 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T07:32:34,776 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732001554771"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001554771"}]},"ts":"1732001554771"}
2024-11-19T07:32:34,782 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T07:32:34,785 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T07:32:34,787 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001554785"}]},"ts":"1732001554785"}
2024-11-19T07:32:34,793 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-19T07:32:34,795 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=33cb0d40cd2d185d3207eca50152e3cc, ASSIGN}]
2024-11-19T07:32:34,798 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=33cb0d40cd2d185d3207eca50152e3cc, ASSIGN
2024-11-19T07:32:34,800 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=33cb0d40cd2d185d3207eca50152e3cc, ASSIGN; state=OFFLINE, location=db0afee3eab9,34769,1732001551610; forceNewPlan=false, retain=false
2024-11-19T07:32:34,952 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33cb0d40cd2d185d3207eca50152e3cc, regionState=OPENING, regionLocation=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:34,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=33cb0d40cd2d185d3207eca50152e3cc, ASSIGN because future has completed
2024-11-19T07:32:34,959 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33cb0d40cd2d185d3207eca50152e3cc, server=db0afee3eab9,34769,1732001551610}]
2024-11-19T07:32:35,122 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:35,122 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 33cb0d40cd2d185d3207eca50152e3cc, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:32:35,123 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,123 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:32:35,123 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,123 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,128 INFO  [StoreOpener-33cb0d40cd2d185d3207eca50152e3cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 33cb0d40cd2d185d3207eca50152e3cc 
2024-11-19T07:32:35,132 INFO  [StoreOpener-33cb0d40cd2d185d3207eca50152e3cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33cb0d40cd2d185d3207eca50152e3cc columnFamilyName info
2024-11-19T07:32:35,132 DEBUG [StoreOpener-33cb0d40cd2d185d3207eca50152e3cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:32:35,133 INFO  [StoreOpener-33cb0d40cd2d185d3207eca50152e3cc-1 {}] regionserver.HStore(327): Store=33cb0d40cd2d185d3207eca50152e3cc/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:32:35,133 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,135 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,135 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,136 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,136 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,139 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,143 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:32:35,145 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 33cb0d40cd2d185d3207eca50152e3cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791164, jitterRate=0.006017521023750305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:32:35,145 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:35,146 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 33cb0d40cd2d185d3207eca50152e3cc:
Running coprocessor pre-open hook at 1732001555123Writing region info on filesystem at 1732001555123Initializing all the Stores at 1732001555125 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001555125Cleaning up temporary data from old regions at 1732001555136 (+11 ms)Running coprocessor post-open hooks at 1732001555145 (+9 ms)Region opened successfully at 1732001555146 (+1 ms)
2024-11-19T07:32:35,148 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc., pid=6, masterSystemTime=1732001555114
2024-11-19T07:32:35,153 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:35,153 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:35,154 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33cb0d40cd2d185d3207eca50152e3cc, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,34769,1732001551610
2024-11-19T07:32:35,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33cb0d40cd2d185d3207eca50152e3cc, server=db0afee3eab9,34769,1732001551610 because future has completed
2024-11-19T07:32:35,166 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T07:32:35,168 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 33cb0d40cd2d185d3207eca50152e3cc, server=db0afee3eab9,34769,1732001551610 in 202 msec
2024-11-19T07:32:35,172 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T07:32:35,173 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=33cb0d40cd2d185d3207eca50152e3cc, ASSIGN in 371 msec
2024-11-19T07:32:35,175 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T07:32:35,175 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001555175"}]},"ts":"1732001555175"}
2024-11-19T07:32:35,185 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-19T07:32:35,188 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T07:32:35,193 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 511 msec
2024-11-19T07:32:39,710 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-19T07:32:39,760 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:32:39,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-19T07:32:41,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:32:41,079 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T07:32:41,082 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-19T07:32:41,082 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T07:32:41,084 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:32:41,084 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T07:32:41,085 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:32:41,085 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T07:32:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44713 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:32:44,770 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-19T07:32:44,772 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-19T07:32:44,780 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-19T07:32:44,780 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:32:44,781 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001564781
2024-11-19T07:32:44,790 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:32:44,791 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:32:44,791 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:32:44,791 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:32:44,791 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:32:44,792 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001553761 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001564781
2024-11-19T07:32:44,793 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:36895:36895)]
2024-11-19T07:32:44,794 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001553761 is not closed yet, will try archiving it next time
2024-11-19T07:32:44,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741833_1009 (size=451)
2024-11-19T07:32:44,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741833_1009 (size=451)
2024-11-19T07:32:44,797 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001553761 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001553761
2024-11-19T07:32:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc., hostname=db0afee3eab9,34769,1732001551610, seqNum=2]
2024-11-19T07:32:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34769 {}] regionserver.HRegion(8855): Flush requested on 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:32:56,850 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33cb0d40cd2d185d3207eca50152e3cc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:32:56,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/da6f1b7ba93f4e38b814b84a06bac42a is 1080, key is row0001/info:/1732001564807/Put/seqid=0
2024-11-19T07:32:56,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741838_1014 (size=12509)
2024-11-19T07:32:56,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741838_1014 (size=12509)
2024-11-19T07:32:56,955 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/da6f1b7ba93f4e38b814b84a06bac42a
2024-11-19T07:32:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/da6f1b7ba93f4e38b814b84a06bac42a as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a
2024-11-19T07:32:57,028 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a, entries=7, sequenceid=11, filesize=12.2 K
2024-11-19T07:32:57,037 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 185ms, sequenceid=11, compaction requested=false
2024-11-19T07:32:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33cb0d40cd2d185d3207eca50152e3cc:

2024-11-19T07:32:59,840 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:33:04,858 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001584858
2024-11-19T07:33:05,067 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:05,067 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:05,067 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:05,067 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:05,068 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:05,068 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:05,068 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001564781 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001584858
2024-11-19T07:33:05,071 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:36895:36895)]
2024-11-19T07:33:05,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741837_1013 (size=12399)
2024-11-19T07:33:05,072 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001564781 is not closed yet, will try archiving it next time
2024-11-19T07:33:05,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741837_1013 (size=12399)
2024-11-19T07:33:05,275 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:07,479 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:09,683 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:11,887 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34769 {}] regionserver.HRegion(8855): Flush requested on 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:33:11,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33cb0d40cd2d185d3207eca50152e3cc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:33:12,090 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:12,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/164a8783798d42279b699445f8b89092 is 1080, key is row0008/info:/1732001578848/Put/seqid=0
2024-11-19T07:33:12,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741840_1016 (size=12509)
2024-11-19T07:33:12,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741840_1016 (size=12509)
2024-11-19T07:33:12,111 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/164a8783798d42279b699445f8b89092
2024-11-19T07:33:12,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/164a8783798d42279b699445f8b89092 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092
2024-11-19T07:33:12,143 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092, entries=7, sequenceid=21, filesize=12.2 K
2024-11-19T07:33:12,345 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:12,345 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 458ms, sequenceid=21, compaction requested=false
2024-11-19T07:33:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33cb0d40cd2d185d3207eca50152e3cc:

2024-11-19T07:33:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-19T07:33:12,345 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:12,346 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a because midkey is the same as first or last row
2024-11-19T07:33:14,091 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:14,961 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T07:33:14,961 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T07:33:16,295 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:16,298 WARN  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:16,299 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C34769%2C1732001551610:(num 1732001584858) roll requested
2024-11-19T07:33:16,299 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001596299
2024-11-19T07:33:16,507 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:16,508 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:16,508 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:16,508 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:16,508 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:16,508 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:16,508 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001584858 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001596299
2024-11-19T07:33:16,509 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:33:16,509 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001584858 is not closed yet, will try archiving it next time
2024-11-19T07:33:16,509 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001564781 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001564781
2024-11-19T07:33:16,511 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741839_1015 (size=7739)
2024-11-19T07:33:16,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741839_1015 (size=7739)
2024-11-19T07:33:18,499 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:20,123 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 33cb0d40cd2d185d3207eca50152e3cc, had cached 0 bytes from a total of 25018
2024-11-19T07:33:20,703 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:22,909 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:25,118 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:27,124 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T07:33:27,124 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001607124
2024-11-19T07:33:29,841 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:33:32,139 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:32,141 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:32,142 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C34769%2C1732001551610:(num 1732001607124) roll requested
2024-11-19T07:33:32,142 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:32,142 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:32,142 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:32,142 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:32,142 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:32,143 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001596299 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001607124
2024-11-19T07:33:32,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:36895:36895)]
2024-11-19T07:33:32,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001596299 is not closed yet, will try archiving it next time
2024-11-19T07:33:32,145 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001612144
2024-11-19T07:33:32,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741841_1017 (size=4753)
2024-11-19T07:33:32,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741841_1017 (size=4753)
2024-11-19T07:33:37,148 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:37,149 WARN  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34769 {}] regionserver.HRegion(8855): Flush requested on 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:33:37,150 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33cb0d40cd2d185d3207eca50152e3cc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:33:37,159 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:37,159 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:39,150 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T07:33:42,153 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:42,153 WARN  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK], DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK]]
2024-11-19T07:33:42,153 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:42,153 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:42,154 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:42,154 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:42,154 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:42,154 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001607124 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001612144
2024-11-19T07:33:42,156 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:33:42,156 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001607124 is not closed yet, will try archiving it next time
2024-11-19T07:33:42,156 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C34769%2C1732001551610:(num 1732001612144) roll requested
2024-11-19T07:33:42,157 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001622157
2024-11-19T07:33:42,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741842_1018 (size=1569)
2024-11-19T07:33:42,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741842_1018 (size=1569)
2024-11-19T07:33:42,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/0411f9fdfbc14093a8c8c2a69d26f561 is 1080, key is row0015/info:/1732001593890/Put/seqid=0
2024-11-19T07:33:42,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741844_1020 (size=12509)
2024-11-19T07:33:42,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741844_1020 (size=12509)
2024-11-19T07:33:42,173 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/0411f9fdfbc14093a8c8c2a69d26f561
2024-11-19T07:33:42,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/0411f9fdfbc14093a8c8c2a69d26f561 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561
2024-11-19T07:33:42,192 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561, entries=7, sequenceid=31, filesize=12.2 K
2024-11-19T07:33:47,170 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:47,171 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:47,193 INFO  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:47,193 WARN  [FSHLog-0-hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b-prefix:db0afee3eab9,34769,1732001551610 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34019,DS-6c95c2a4-9cdd-4d77-9914-e04264ec1942,DISK], DatanodeInfoWithStorage[127.0.0.1:34237,DS-aedb410d-3b10-4698-a67d-0328ef4006a9,DISK]]
2024-11-19T07:33:47,193 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 10044ms, sequenceid=31, compaction requested=true
2024-11-19T07:33:47,194 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33cb0d40cd2d185d3207eca50152e3cc:

2024-11-19T07:33:47,194 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,194 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-19T07:33:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:47,194 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a because midkey is the same as first or last row
2024-11-19T07:33:47,194 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,194 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001612144 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001622157
2024-11-19T07:33:47,195 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:33:47,195 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001612144 is not closed yet, will try archiving it next time
2024-11-19T07:33:47,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33cb0d40cd2d185d3207eca50152e3cc:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:33:47,195 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001584858 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001584858
2024-11-19T07:33:47,195 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C34769%2C1732001551610:(num 1732001627195) roll requested
2024-11-19T07:33:47,196 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001627195
2024-11-19T07:33:47,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741843_1019 (size=438)
2024-11-19T07:33:47,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:33:47,198 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001596299 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001596299
2024-11-19T07:33:47,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741843_1019 (size=438)
2024-11-19T07:33:47,198 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:33:47,199 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001607124 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001607124
2024-11-19T07:33:47,201 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:33:47,201 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001612144 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001612144
2024-11-19T07:33:47,202 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HStore(1541): 33cb0d40cd2d185d3207eca50152e3cc/info is initiating minor compaction (all files)
2024-11-19T07:33:47,203 INFO  [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33cb0d40cd2d185d3207eca50152e3cc/info in TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:33:47,203 INFO  [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561] into tmpdir=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp, totalSize=36.6 K
2024-11-19T07:33:47,205 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6f1b7ba93f4e38b814b84a06bac42a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732001564807
2024-11-19T07:33:47,205 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] compactions.Compactor(225): Compacting 164a8783798d42279b699445f8b89092, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732001578848
2024-11-19T07:33:47,206 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0411f9fdfbc14093a8c8c2a69d26f561, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732001593890
2024-11-19T07:33:47,210 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,210 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,210 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,210 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,210 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,211 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001622157 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001627195
2024-11-19T07:33:47,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:33:47,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001622157 is not closed yet, will try archiving it next time
2024-11-19T07:33:47,212 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C34769%2C1732001551610.1732001627212
2024-11-19T07:33:47,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741845_1021 (size=93)
2024-11-19T07:33:47,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741845_1021 (size=93)
2024-11-19T07:33:47,215 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001622157 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs/db0afee3eab9%2C34769%2C1732001551610.1732001622157
2024-11-19T07:33:47,233 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,233 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,233 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,233 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,234 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:33:47,234 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001627195 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001627212
2024-11-19T07:33:47,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741846_1022 (size=1258)
2024-11-19T07:33:47,237 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36895:36895),(127.0.0.1/127.0.0.1:36067:36067)]
2024-11-19T07:33:47,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741846_1022 (size=1258)
2024-11-19T07:33:47,237 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/WALs/db0afee3eab9,34769,1732001551610/db0afee3eab9%2C34769%2C1732001551610.1732001627195 is not closed yet, will try archiving it next time
2024-11-19T07:33:47,245 INFO  [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33cb0d40cd2d185d3207eca50152e3cc#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:33:47,247 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/f5ba4c34d0ff47c89890aede1c564cbc is 1080, key is row0001/info:/1732001564807/Put/seqid=0
2024-11-19T07:33:47,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741848_1024 (size=27710)
2024-11-19T07:33:47,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741848_1024 (size=27710)
2024-11-19T07:33:47,265 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/f5ba4c34d0ff47c89890aede1c564cbc as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f5ba4c34d0ff47c89890aede1c564cbc
2024-11-19T07:33:47,280 INFO  [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33cb0d40cd2d185d3207eca50152e3cc/info of 33cb0d40cd2d185d3207eca50152e3cc into f5ba4c34d0ff47c89890aede1c564cbc(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:33:47,280 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33cb0d40cd2d185d3207eca50152e3cc:

2024-11-19T07:33:47,282 INFO  [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc., storeName=33cb0d40cd2d185d3207eca50152e3cc/info, priority=13, startTime=1732001627195; duration=0sec
2024-11-19T07:33:47,282 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T07:33:47,282 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:47,282 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f5ba4c34d0ff47c89890aede1c564cbc because midkey is the same as first or last row
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f5ba4c34d0ff47c89890aede1c564cbc because midkey is the same as first or last row
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f5ba4c34d0ff47c89890aede1c564cbc because midkey is the same as first or last row
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:33:47,283 DEBUG [RS:0;db0afee3eab9:34769-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33cb0d40cd2d185d3207eca50152e3cc:info
2024-11-19T07:33:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34769 {}] regionserver.HRegion(8855): Flush requested on 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:33:59,251 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33cb0d40cd2d185d3207eca50152e3cc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:33:59,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/f07b61ae39d74cb6aa1e2c54f9fe1278 is 1080, key is row0022/info:/1732001627214/Put/seqid=0
2024-11-19T07:33:59,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741849_1025 (size=12509)
2024-11-19T07:33:59,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741849_1025 (size=12509)
2024-11-19T07:33:59,270 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/f07b61ae39d74cb6aa1e2c54f9fe1278
2024-11-19T07:33:59,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/f07b61ae39d74cb6aa1e2c54f9fe1278 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f07b61ae39d74cb6aa1e2c54f9fe1278
2024-11-19T07:33:59,294 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f07b61ae39d74cb6aa1e2c54f9fe1278, entries=7, sequenceid=42, filesize=12.2 K
2024-11-19T07:33:59,296 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 45ms, sequenceid=42, compaction requested=false
2024-11-19T07:33:59,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33cb0d40cd2d185d3207eca50152e3cc:

2024-11-19T07:33:59,296 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-19T07:33:59,296 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:33:59,296 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/f5ba4c34d0ff47c89890aede1c564cbc because midkey is the same as first or last row
2024-11-19T07:33:59,841 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:34:05,124 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 33cb0d40cd2d185d3207eca50152e3cc, had cached 0 bytes from a total of 40219
2024-11-19T07:34:07,263 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:34:07,263 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:07,264 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:07,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:07,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:07,273 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:34:07,273 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:34:07,273 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1407995813, stopped=false
2024-11-19T07:34:07,274 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,44713,1732001550692
2024-11-19T07:34:07,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:07,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:07,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:07,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:07,313 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:34:07,313 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:07,313 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:07,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:07,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:07,313 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:07,314 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,34769,1732001551610' *****
2024-11-19T07:34:07,314 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:34:07,314 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:34:07,314 INFO  [RS:0;db0afee3eab9:34769 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:34:07,314 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:34:07,314 INFO  [RS:0;db0afee3eab9:34769 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(3091): Received CLOSE for 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,34769,1732001551610
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:34769.
2024-11-19T07:34:07,315 DEBUG [RS:0;db0afee3eab9:34769 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:07,315 DEBUG [RS:0;db0afee3eab9:34769 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:34:07,315 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33cb0d40cd2d185d3207eca50152e3cc, disabling compactions & flushes
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:34:07,315 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:34:07,315 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:34:07,316 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc. after waiting 0 ms
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:34:07,316 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T07:34:07,316 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 33cb0d40cd2d185d3207eca50152e3cc 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-19T07:34:07,316 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 33cb0d40cd2d185d3207eca50152e3cc=TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.}
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:34:07,316 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:34:07,316 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:34:07,316 DEBUG [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 33cb0d40cd2d185d3207eca50152e3cc
2024-11-19T07:34:07,316 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-19T07:34:07,321 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/39f855cd0b134d6f8ecf34f182405379 is 1080, key is row0029/info:/1732001641253/Put/seqid=0
2024-11-19T07:34:07,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741850_1026 (size=8193)
2024-11-19T07:34:07,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741850_1026 (size=8193)
2024-11-19T07:34:07,328 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/39f855cd0b134d6f8ecf34f182405379
2024-11-19T07:34:07,338 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/.tmp/info/39f855cd0b134d6f8ecf34f182405379 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/39f855cd0b134d6f8ecf34f182405379
2024-11-19T07:34:07,341 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/info/740c7f41c7c44903999d4d5e0e44f681 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc./info:regioninfo/1732001555154/Put/seqid=0
2024-11-19T07:34:07,346 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/39f855cd0b134d6f8ecf34f182405379, entries=3, sequenceid=48, filesize=8.0 K
2024-11-19T07:34:07,348 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 32ms, sequenceid=48, compaction requested=true
2024-11-19T07:34:07,351 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561] to archive
2024-11-19T07:34:07,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741851_1027 (size=7016)
2024-11-19T07:34:07,354 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:34:07,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741851_1027 (size=7016)
2024-11-19T07:34:07,356 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/info/740c7f41c7c44903999d4d5e0e44f681
2024-11-19T07:34:07,358 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/da6f1b7ba93f4e38b814b84a06bac42a
2024-11-19T07:34:07,360 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/164a8783798d42279b699445f8b89092
2024-11-19T07:34:07,362 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561 to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/info/0411f9fdfbc14093a8c8c2a69d26f561
2024-11-19T07:34:07,373 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:44713 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-19T07:34:07,377 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [da6f1b7ba93f4e38b814b84a06bac42a=12509, 164a8783798d42279b699445f8b89092=12509, 0411f9fdfbc14093a8c8c2a69d26f561=12509]
2024-11-19T07:34:07,382 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/default/TestLogRolling-testSlowSyncLogRolling/33cb0d40cd2d185d3207eca50152e3cc/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-19T07:34:07,385 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:34:07,385 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33cb0d40cd2d185d3207eca50152e3cc:
Waiting for close lock at 1732001647315Running coprocessor pre-close hooks at 1732001647315Disabling compacts and flushes for region at 1732001647315Disabling writes for close at 1732001647316 (+1 ms)Obtaining lock to block concurrent updates at 1732001647316Preparing flush snapshotting stores in 33cb0d40cd2d185d3207eca50152e3cc at 1732001647316Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732001647316Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc. at 1732001647317 (+1 ms)Flushing 33cb0d40cd2d185d3207eca50152e3cc/info: creating writer at 1732001647317Flushing 33cb0d40cd2d185d3207eca50152e3cc/info: appending metadata at 1732001647321 (+4 ms)Flushing 33cb0d40cd2d185d3207eca50152e3cc/info: closing flushed file at 1732001647321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bfa4c8d: reopening flushed file at 1732001647337 (+16 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 33cb0d40cd2d185d3207eca50152e3cc in 32ms, sequenceid=48, compaction requested=true at 1732001647348 (+11 ms)Writing region close event to WAL at 1732001647377 (+29 ms)Running coprocessor post-close hooks at 1732001647383 (+6 ms)Closed at 1732001647385 (+2 ms)
2024-11-19T07:34:07,385 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732001554668.33cb0d40cd2d185d3207eca50152e3cc.
2024-11-19T07:34:07,387 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/ns/be3053ff2894406a8be1769e5409eb43 is 43, key is default/ns:d/1732001554416/Put/seqid=0
2024-11-19T07:34:07,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741852_1028 (size=5153)
2024-11-19T07:34:07,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741852_1028 (size=5153)
2024-11-19T07:34:07,394 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/ns/be3053ff2894406a8be1769e5409eb43
2024-11-19T07:34:07,425 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/table/d7e2585f23254efca611cca3a91a8219 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732001555175/Put/seqid=0
2024-11-19T07:34:07,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741853_1029 (size=5396)
2024-11-19T07:34:07,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741853_1029 (size=5396)
2024-11-19T07:34:07,431 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/table/d7e2585f23254efca611cca3a91a8219
2024-11-19T07:34:07,440 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/info/740c7f41c7c44903999d4d5e0e44f681 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/info/740c7f41c7c44903999d4d5e0e44f681
2024-11-19T07:34:07,449 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/info/740c7f41c7c44903999d4d5e0e44f681, entries=10, sequenceid=11, filesize=6.9 K
2024-11-19T07:34:07,451 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/ns/be3053ff2894406a8be1769e5409eb43 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/ns/be3053ff2894406a8be1769e5409eb43
2024-11-19T07:34:07,461 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/ns/be3053ff2894406a8be1769e5409eb43, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T07:34:07,463 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/.tmp/table/d7e2585f23254efca611cca3a91a8219 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/table/d7e2585f23254efca611cca3a91a8219
2024-11-19T07:34:07,472 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/table/d7e2585f23254efca611cca3a91a8219, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T07:34:07,474 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 157ms, sequenceid=11, compaction requested=false
2024-11-19T07:34:07,479 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T07:34:07,480 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:34:07,480 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:07,481 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001647316Running coprocessor pre-close hooks at 1732001647316Disabling compacts and flushes for region at 1732001647316Disabling writes for close at 1732001647316Obtaining lock to block concurrent updates at 1732001647316Preparing flush snapshotting stores in 1588230740 at 1732001647316Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732001647317 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732001647317Flushing 1588230740/info: creating writer at 1732001647318 (+1 ms)Flushing 1588230740/info: appending metadata at 1732001647341 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732001647341Flushing 1588230740/ns: creating writer at 1732001647366 (+25 ms)Flushing 1588230740/ns: appending metadata at 1732001647387 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732001647387Flushing 1588230740/table: creating writer at 1732001647403 (+16 ms)Flushing 1588230740/table: appending metadata at 1732001647424 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732001647424Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cbaed: reopening flushed file at 1732001647439 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b302424: reopening flushed file at 1732001647450 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eaa2ff6: reopening flushed file at 1732001647462 (+12 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 157ms, sequenceid=11, compaction requested=false at 1732001647474 (+12 ms)Writing region close event to WAL at 1732001647475 (+1 ms)Running coprocessor post-close hooks at 1732001647480 (+5 ms)Closed at 1732001647480
2024-11-19T07:34:07,481 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:07,517 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,34769,1732001551610; all regions closed.
2024-11-19T07:34:07,519 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,519 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,520 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,520 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,520 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741834_1010 (size=3066)
2024-11-19T07:34:07,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741834_1010 (size=3066)
2024-11-19T07:34:07,529 DEBUG [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs
2024-11-19T07:34:07,529 INFO  [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C34769%2C1732001551610.meta:.meta(num 1732001554168)
2024-11-19T07:34:07,530 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,530 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,530 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,531 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,531 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741847_1023 (size=12695)
2024-11-19T07:34:07,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741847_1023 (size=12695)
2024-11-19T07:34:07,538 DEBUG [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/oldWALs
2024-11-19T07:34:07,538 INFO  [RS:0;db0afee3eab9:34769 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C34769%2C1732001551610:(num 1732001627212)
2024-11-19T07:34:07,538 DEBUG [RS:0;db0afee3eab9:34769 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:07,538 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:07,538 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:34:07,538 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:34:07,538 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:34:07,538 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:34:07,539 INFO  [RS:0;db0afee3eab9:34769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34769
2024-11-19T07:34:07,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:07,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,34769,1732001551610
2024-11-19T07:34:07,548 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:34:07,559 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,34769,1732001551610]
2024-11-19T07:34:07,569 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,34769,1732001551610 already deleted, retry=false
2024-11-19T07:34:07,569 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,34769,1732001551610 expired; onlineServers=0
2024-11-19T07:34:07,569 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,44713,1732001550692' *****
2024-11-19T07:34:07,569 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:34:07,569 INFO  [M:0;db0afee3eab9:44713 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:07,570 INFO  [M:0;db0afee3eab9:44713 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:34:07,570 DEBUG [M:0;db0afee3eab9:44713 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:34:07,570 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:34:07,570 DEBUG [M:0;db0afee3eab9:44713 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:34:07,570 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001553371 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001553371,5,FailOnTimeoutGroup]
2024-11-19T07:34:07,570 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001553373 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001553373,5,FailOnTimeoutGroup]
2024-11-19T07:34:07,570 INFO  [M:0;db0afee3eab9:44713 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:34:07,570 INFO  [M:0;db0afee3eab9:44713 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:34:07,570 DEBUG [M:0;db0afee3eab9:44713 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:34:07,570 INFO  [M:0;db0afee3eab9:44713 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:34:07,571 INFO  [M:0;db0afee3eab9:44713 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:34:07,571 INFO  [M:0;db0afee3eab9:44713 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:34:07,571 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:34:07,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:34:07,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:07,580 DEBUG [M:0;db0afee3eab9:44713 {}] zookeeper.ZKUtil(347): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:34:07,580 WARN  [M:0;db0afee3eab9:44713 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:34:07,581 INFO  [M:0;db0afee3eab9:44713 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/.lastflushedseqids
2024-11-19T07:34:07,594 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741854_1030 (size=130)
2024-11-19T07:34:07,594 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741854_1030 (size=130)
2024-11-19T07:34:07,595 INFO  [M:0;db0afee3eab9:44713 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:34:07,595 INFO  [M:0;db0afee3eab9:44713 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:34:07,595 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:34:07,595 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:07,595 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:07,595 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:34:07,595 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:07,596 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-19T07:34:07,609 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:07,616 DEBUG [M:0;db0afee3eab9:44713 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/96d186c40e2646cebecaba19b6bfe895 is 82, key is hbase:meta,,1/info:regioninfo/1732001554248/Put/seqid=0
2024-11-19T07:34:07,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741855_1031 (size=5672)
2024-11-19T07:34:07,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741855_1031 (size=5672)
2024-11-19T07:34:07,623 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/96d186c40e2646cebecaba19b6bfe895
2024-11-19T07:34:07,647 DEBUG [M:0;db0afee3eab9:44713 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f61dd02e630742379c0dbd930980ca24 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732001555191/Put/seqid=0
2024-11-19T07:34:07,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741856_1032 (size=6248)
2024-11-19T07:34:07,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741856_1032 (size=6248)
2024-11-19T07:34:07,653 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f61dd02e630742379c0dbd930980ca24
2024-11-19T07:34:07,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:07,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34769-0x101520249ee0001, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:07,659 INFO  [RS:0;db0afee3eab9:34769 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:34:07,659 INFO  [RS:0;db0afee3eab9:34769 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,34769,1732001551610; zookeeper connection closed.
2024-11-19T07:34:07,660 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f61dd02e630742379c0dbd930980ca24
2024-11-19T07:34:07,660 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b98826f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b98826f
2024-11-19T07:34:07,660 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:34:07,677 DEBUG [M:0;db0afee3eab9:44713 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/934258a0b6f14dfe9834e1768809554a is 69, key is db0afee3eab9,34769,1732001551610/rs:state/1732001553522/Put/seqid=0
2024-11-19T07:34:07,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741857_1033 (size=5156)
2024-11-19T07:34:07,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741857_1033 (size=5156)
2024-11-19T07:34:07,683 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/934258a0b6f14dfe9834e1768809554a
2024-11-19T07:34:07,707 DEBUG [M:0;db0afee3eab9:44713 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b52e3d57b504628a168497991ca0112 is 52, key is load_balancer_on/state:d/1732001554629/Put/seqid=0
2024-11-19T07:34:07,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741858_1034 (size=5056)
2024-11-19T07:34:07,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741858_1034 (size=5056)
2024-11-19T07:34:07,714 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b52e3d57b504628a168497991ca0112
2024-11-19T07:34:07,723 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/96d186c40e2646cebecaba19b6bfe895 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/96d186c40e2646cebecaba19b6bfe895
2024-11-19T07:34:07,731 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/96d186c40e2646cebecaba19b6bfe895, entries=8, sequenceid=59, filesize=5.5 K
2024-11-19T07:34:07,733 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f61dd02e630742379c0dbd930980ca24 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f61dd02e630742379c0dbd930980ca24
2024-11-19T07:34:07,740 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f61dd02e630742379c0dbd930980ca24
2024-11-19T07:34:07,740 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f61dd02e630742379c0dbd930980ca24, entries=6, sequenceid=59, filesize=6.1 K
2024-11-19T07:34:07,741 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/934258a0b6f14dfe9834e1768809554a as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/934258a0b6f14dfe9834e1768809554a
2024-11-19T07:34:07,748 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/934258a0b6f14dfe9834e1768809554a, entries=1, sequenceid=59, filesize=5.0 K
2024-11-19T07:34:07,750 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5b52e3d57b504628a168497991ca0112 as hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5b52e3d57b504628a168497991ca0112
2024-11-19T07:34:07,757 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5b52e3d57b504628a168497991ca0112, entries=1, sequenceid=59, filesize=4.9 K
2024-11-19T07:34:07,759 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=59, compaction requested=false
2024-11-19T07:34:07,761 INFO  [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:07,761 DEBUG [M:0;db0afee3eab9:44713 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001647595Disabling compacts and flushes for region at 1732001647595Disabling writes for close at 1732001647595Obtaining lock to block concurrent updates at 1732001647596 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001647596Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1732001647596Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001647597 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001647598 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001647616 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001647616Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001647630 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001647646 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001647646Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001647660 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001647676 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001647676Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001647691 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001647706 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001647706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734b0408: reopening flushed file at 1732001647722 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@768e687: reopening flushed file at 1732001647731 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@515959f3: reopening flushed file at 1732001647740 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4447e9c0: reopening flushed file at 1732001647749 (+9 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=59, compaction requested=false at 1732001647759 (+10 ms)Writing region close event to WAL at 1732001647761 (+2 ms)Closed at 1732001647761
2024-11-19T07:34:07,762 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,762 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,762 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,762 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,762 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:07,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34237 is added to blk_1073741830_1006 (size=27985)
2024-11-19T07:34:07,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34019 is added to blk_1073741830_1006 (size=27985)
2024-11-19T07:34:07,766 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:34:07,766 INFO  [M:0;db0afee3eab9:44713 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:34:07,766 INFO  [M:0;db0afee3eab9:44713 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44713
2024-11-19T07:34:07,766 INFO  [M:0;db0afee3eab9:44713 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:34:07,890 INFO  [M:0;db0afee3eab9:44713 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:34:07,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:07,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44713-0x101520249ee0000, quorum=127.0.0.1:53123, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:07,898 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:07,901 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:07,901 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:07,901 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:07,901 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:07,904 WARN  [BP-2111989087-172.17.0.2-1732001546043 heartbeating to localhost/127.0.0.1:45255 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:07,904 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:07,904 WARN  [BP-2111989087-172.17.0.2-1732001546043 heartbeating to localhost/127.0.0.1:45255 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2111989087-172.17.0.2-1732001546043 (Datanode Uuid 596598a2-c950-4c83-8fa3-964c717d5c3d) service to localhost/127.0.0.1:45255
2024-11-19T07:34:07,904 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:07,905 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data3/current/BP-2111989087-172.17.0.2-1732001546043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:07,906 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data4/current/BP-2111989087-172.17.0.2-1732001546043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:07,906 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:07,909 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:07,912 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:07,913 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:07,913 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:07,913 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:07,915 WARN  [BP-2111989087-172.17.0.2-1732001546043 heartbeating to localhost/127.0.0.1:45255 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:07,915 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:07,915 WARN  [BP-2111989087-172.17.0.2-1732001546043 heartbeating to localhost/127.0.0.1:45255 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2111989087-172.17.0.2-1732001546043 (Datanode Uuid 538eaccd-ca9b-4a2a-a614-9beed8fe8f44) service to localhost/127.0.0.1:45255
2024-11-19T07:34:07,915 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:07,916 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data1/current/BP-2111989087-172.17.0.2-1732001546043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:07,916 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/cluster_1fe1102e-0b29-829e-4fab-8b90864b2453/data/data2/current/BP-2111989087-172.17.0.2-1732001546043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:07,917 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:07,930 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:34:07,930 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:07,930 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:07,931 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:07,931 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:07,939 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:34:07,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:34:07,991 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45255 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:45255
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45255 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: master/db0afee3eab9:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@7b72834e
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45255 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45255
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/db0afee3eab9:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/db0afee3eab9:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=156 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=12504 (was 13082)
2024-11-19T07:34:07,998 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=156, ProcessCount=11, AvailableMemoryMB=12504
2024-11-19T07:34:07,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:34:07,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.log.dir so I do NOT create it in target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd
2024-11-19T07:34:07,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/55b4db71-81c5-2bfa-17fa-0666f304db01/hadoop.tmp.dir so I do NOT create it in target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39, deleteOnExit=true
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/test.cache.data in system properties and HBase conf
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:34:07,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:34:07,999 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:34:08,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:34:08,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:34:08,014 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:34:08,489 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:08,498 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:08,504 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:08,504 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:08,504 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:08,505 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:08,506 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65506a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:08,506 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75cbfab9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:08,602 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@493d1d34{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/java.io.tmpdir/jetty-localhost-42751-hadoop-hdfs-3_4_1-tests_jar-_-any-6669663812741391842/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:34:08,602 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6a249094{HTTP/1.1, (http/1.1)}{localhost:42751}
2024-11-19T07:34:08,603 INFO  [Time-limited test {}] server.Server(415): Started @105291ms
2024-11-19T07:34:08,614 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:34:08,881 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:08,885 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:08,886 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:08,886 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:08,886 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:08,886 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c8914e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:08,887 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aa9354f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:08,991 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@52b07bdb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/java.io.tmpdir/jetty-localhost-38917-hadoop-hdfs-3_4_1-tests_jar-_-any-11745650160861132053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:08,991 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70121b28{HTTP/1.1, (http/1.1)}{localhost:38917}
2024-11-19T07:34:08,991 INFO  [Time-limited test {}] server.Server(415): Started @105680ms
2024-11-19T07:34:08,993 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:09,054 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:09,060 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:09,061 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:09,061 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:09,061 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:34:09,062 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b915b67{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:09,062 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc8c098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:09,162 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f5c23ef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/java.io.tmpdir/jetty-localhost-33885-hadoop-hdfs-3_4_1-tests_jar-_-any-15391573806505714426/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:09,163 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@716c7b87{HTTP/1.1, (http/1.1)}{localhost:33885}
2024-11-19T07:34:09,163 INFO  [Time-limited test {}] server.Server(415): Started @105851ms
2024-11-19T07:34:09,164 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:10,264 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data1/current/BP-1712531913-172.17.0.2-1732001648026/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:10,264 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data2/current/BP-1712531913-172.17.0.2-1732001648026/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:10,284 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:10,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9b734bebe8a0182 with lease ID 0xb614d756dc0ef037: Processing first storage report for DS-7253e35f-d606-4e47-93e1-73507dc46fb2 from datanode DatanodeRegistration(127.0.0.1:34021, datanodeUuid=792ddadc-f6f2-45e3-84a8-7132247f4c12, infoPort=36597, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026)
2024-11-19T07:34:10,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9b734bebe8a0182 with lease ID 0xb614d756dc0ef037: from storage DS-7253e35f-d606-4e47-93e1-73507dc46fb2 node DatanodeRegistration(127.0.0.1:34021, datanodeUuid=792ddadc-f6f2-45e3-84a8-7132247f4c12, infoPort=36597, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:10,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9b734bebe8a0182 with lease ID 0xb614d756dc0ef037: Processing first storage report for DS-e548e245-bbb8-42c1-9b29-1deb0acf7f41 from datanode DatanodeRegistration(127.0.0.1:34021, datanodeUuid=792ddadc-f6f2-45e3-84a8-7132247f4c12, infoPort=36597, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026)
2024-11-19T07:34:10,287 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9b734bebe8a0182 with lease ID 0xb614d756dc0ef037: from storage DS-e548e245-bbb8-42c1-9b29-1deb0acf7f41 node DatanodeRegistration(127.0.0.1:34021, datanodeUuid=792ddadc-f6f2-45e3-84a8-7132247f4c12, infoPort=36597, infoSecurePort=0, ipcPort=42321, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:10,489 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data4/current/BP-1712531913-172.17.0.2-1732001648026/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:10,489 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data3/current/BP-1712531913-172.17.0.2-1732001648026/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:10,507 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:10,509 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5270e5f6acbd976 with lease ID 0xb614d756dc0ef038: Processing first storage report for DS-577e374d-a41f-44c2-9372-254eea5b4394 from datanode DatanodeRegistration(127.0.0.1:38797, datanodeUuid=29f38b1b-e788-4823-84ac-21efa4e05822, infoPort=42351, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026)
2024-11-19T07:34:10,509 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5270e5f6acbd976 with lease ID 0xb614d756dc0ef038: from storage DS-577e374d-a41f-44c2-9372-254eea5b4394 node DatanodeRegistration(127.0.0.1:38797, datanodeUuid=29f38b1b-e788-4823-84ac-21efa4e05822, infoPort=42351, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:10,509 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5270e5f6acbd976 with lease ID 0xb614d756dc0ef038: Processing first storage report for DS-d6a7693c-314a-49d0-b4db-842135328297 from datanode DatanodeRegistration(127.0.0.1:38797, datanodeUuid=29f38b1b-e788-4823-84ac-21efa4e05822, infoPort=42351, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026)
2024-11-19T07:34:10,509 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5270e5f6acbd976 with lease ID 0xb614d756dc0ef038: from storage DS-d6a7693c-314a-49d0-b4db-842135328297 node DatanodeRegistration(127.0.0.1:38797, datanodeUuid=29f38b1b-e788-4823-84ac-21efa4e05822, infoPort=42351, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1672142551;c=1732001648026), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:10,614 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd
2024-11-19T07:34:10,618 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/zookeeper_0, clientPort=50279, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:34:10,619 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50279
2024-11-19T07:34:10,619 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,621 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:34:10,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:34:10,633 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807 with version=8
2024-11-19T07:34:10,633 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:34:10,635 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:34:10,636 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:34:10,636 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41723
2024-11-19T07:34:10,638 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41723 connecting to ZooKeeper ensemble=127.0.0.1:50279
2024-11-19T07:34:10,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:417230x0, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:34:10,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41723-0x1015203d3650000 connected
2024-11-19T07:34:10,781 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,785 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:10,792 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807, hbase.cluster.distributed=false
2024-11-19T07:34:10,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:34:10,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41723
2024-11-19T07:34:10,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41723
2024-11-19T07:34:10,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41723
2024-11-19T07:34:10,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41723
2024-11-19T07:34:10,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41723
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:34:10,814 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:34:10,815 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38135
2024-11-19T07:34:10,816 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38135 connecting to ZooKeeper ensemble=127.0.0.1:50279
2024-11-19T07:34:10,817 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,819 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381350x0, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:34:10,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381350x0, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:10,833 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38135-0x1015203d3650001 connected
2024-11-19T07:34:10,833 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:34:10,834 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:34:10,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:34:10,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:34:10,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38135
2024-11-19T07:34:10,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38135
2024-11-19T07:34:10,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38135
2024-11-19T07:34:10,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38135
2024-11-19T07:34:10,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38135
2024-11-19T07:34:10,859 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:41723
2024-11-19T07:34:10,860 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,41723,1732001650635
2024-11-19T07:34:10,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:10,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:10,872 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,41723,1732001650635
2024-11-19T07:34:10,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:34:10,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:10,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:10,883 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:34:10,883 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,41723,1732001650635 from backup master directory
2024-11-19T07:34:10,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:10,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,41723,1732001650635
2024-11-19T07:34:10,896 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:34:10,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:10,896 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,41723,1732001650635
2024-11-19T07:34:10,901 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/hbase.id] with ID: b0f07a6a-fa8d-438c-8c95-f3c051e9e44d
2024-11-19T07:34:10,901 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/.tmp/hbase.id
2024-11-19T07:34:10,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:34:10,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:34:10,911 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/.tmp/hbase.id]:[hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/hbase.id]
2024-11-19T07:34:10,927 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:10,928 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:34:10,930 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T07:34:10,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:10,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:10,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:34:10,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:34:10,949 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:34:10,950 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:34:10,950 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:10,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:34:10,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:34:11,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:34:11,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:34:11,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:34:11,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-19T07:34:11,361 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store
2024-11-19T07:34:11,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:34:11,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:34:11,375 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:11,375 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:34:11,375 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:11,376 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:11,376 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:34:11,376 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:11,376 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:11,376 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001651375Disabling compacts and flushes for region at 1732001651375Disabling writes for close at 1732001651376 (+1 ms)Writing region close event to WAL at 1732001651376Closed at 1732001651376
2024-11-19T07:34:11,377 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/.initializing
2024-11-19T07:34:11,378 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/WALs/db0afee3eab9,41723,1732001650635
2024-11-19T07:34:11,381 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C41723%2C1732001650635, suffix=, logDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/WALs/db0afee3eab9,41723,1732001650635, archiveDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/oldWALs, maxLogs=10
2024-11-19T07:34:11,382 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C41723%2C1732001650635.1732001651382
2024-11-19T07:34:11,388 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/WALs/db0afee3eab9,41723,1732001650635/db0afee3eab9%2C41723%2C1732001650635.1732001651382
2024-11-19T07:34:11,391 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42351:42351),(127.0.0.1/127.0.0.1:36597:36597)]
2024-11-19T07:34:11,392 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:34:11,392 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:11,392 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,392 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,394 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:11,396 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:34:11,396 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,397 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:11,397 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:11,399 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:34:11,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,401 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:11,401 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:11,404 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:34:11,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,406 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:11,406 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:11,409 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:34:11,409 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,410 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:11,410 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,411 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,412 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:34:11,416 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:11,418 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:34:11,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807817, jitterRate=0.027193263173103333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:34:11,420 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001651393Initializing all the Stores at 1732001651394 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001651394Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001651394Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001651394Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001651394Cleaning up temporary data from old regions at 1732001651414 (+20 ms)Region opened successfully at 1732001651420 (+6 ms)
2024-11-19T07:34:11,420 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:34:11,424 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756bfceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:34:11,425 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:34:11,425 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:34:11,425 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:34:11,426 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:34:11,426 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:34:11,427 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:34:11,427 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:34:11,430 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:34:11,431 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:34:11,482 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:34:11,482 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:34:11,484 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:34:11,495 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:34:11,496 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:34:11,499 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:34:11,506 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:34:11,507 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:34:11,516 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:34:11,521 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:34:11,534 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:34:11,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:11,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:11,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,550 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,41723,1732001650635, sessionid=0x1015203d3650000, setting cluster-up flag (Was=false)
2024-11-19T07:34:11,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,601 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:34:11,606 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,41723,1732001650635
2024-11-19T07:34:11,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:11,664 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:34:11,669 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,41723,1732001650635
2024-11-19T07:34:11,673 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:34:11,677 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:11,677 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:34:11,678 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:34:11,678 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,41723,1732001650635
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:34:11,680 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:11,680 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:11,680 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:11,680 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:11,681 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:34:11,681 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,681 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:34:11,681 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001681682
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:34:11,682 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:34:11,683 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:34:11,683 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,683 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:11,683 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:34:11,683 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:34:11,683 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:34:11,684 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:34:11,684 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:34:11,684 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:34:11,684 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001651684,5,FailOnTimeoutGroup]
2024-11-19T07:34:11,685 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001651684,5,FailOnTimeoutGroup]
2024-11-19T07:34:11,685 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,685 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:34:11,685 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,685 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,685 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,685 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:34:11,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:34:11,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:34:11,698 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:34:11,699 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807
2024-11-19T07:34:11,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:34:11,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:34:11,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:11,708 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:34:11,710 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:34:11,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,710 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:11,711 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:34:11,712 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:34:11,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,713 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:11,713 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:34:11,715 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:34:11,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,716 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:11,716 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:34:11,717 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:34:11,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:11,718 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:11,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:34:11,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740
2024-11-19T07:34:11,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740
2024-11-19T07:34:11,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:34:11,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:34:11,722 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:34:11,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:34:11,726 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:34:11,726 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752919, jitterRate=-0.042614758014678955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:34:11,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001651707Initializing all the Stores at 1732001651708 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001651708Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001651708Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001651708Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001651708Cleaning up temporary data from old regions at 1732001651721 (+13 ms)Region opened successfully at 1732001651728 (+7 ms)
2024-11-19T07:34:11,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:34:11,728 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:34:11,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:34:11,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:34:11,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:34:11,729 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:11,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001651728Disabling compacts and flushes for region at 1732001651728Disabling writes for close at 1732001651729 (+1 ms)Writing region close event to WAL at 1732001651729Closed at 1732001651729
2024-11-19T07:34:11,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:11,731 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:34:11,731 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:34:11,733 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:34:11,734 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:34:11,748 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(746): ClusterId : b0f07a6a-fa8d-438c-8c95-f3c051e9e44d
2024-11-19T07:34:11,748 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:34:11,759 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:34:11,760 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:34:11,770 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:34:11,770 DEBUG [RS:0;db0afee3eab9:38135 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e116948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:34:11,781 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:38135
2024-11-19T07:34:11,781 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:34:11,781 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:34:11,781 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:34:11,782 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,41723,1732001650635 with port=38135, startcode=1732001650813
2024-11-19T07:34:11,782 DEBUG [RS:0;db0afee3eab9:38135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:34:11,785 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:34:11,785 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41723 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,785 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41723 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,787 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807
2024-11-19T07:34:11,788 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42993
2024-11-19T07:34:11,788 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:34:11,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:11,801 DEBUG [RS:0;db0afee3eab9:38135 {}] zookeeper.ZKUtil(111): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,802 WARN  [RS:0;db0afee3eab9:38135 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:34:11,802 INFO  [RS:0;db0afee3eab9:38135 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:11,802 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/WALs/db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,802 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,38135,1732001650813]
2024-11-19T07:34:11,806 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:34:11,808 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:34:11,809 INFO  [RS:0;db0afee3eab9:38135 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:34:11,809 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,809 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:34:11,810 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:34:11,810 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:34:11,810 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:11,811 DEBUG [RS:0;db0afee3eab9:38135 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:11,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,813 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,38135,1732001650813-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:34:11,827 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:34:11,827 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,38135,1732001650813-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,827 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,828 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.Replication(171): db0afee3eab9,38135,1732001650813 started
2024-11-19T07:34:11,842 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:11,842 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,38135,1732001650813, RpcServer on db0afee3eab9/172.17.0.2:38135, sessionid=0x1015203d3650001
2024-11-19T07:34:11,842 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:34:11,842 DEBUG [RS:0;db0afee3eab9:38135 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,842 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,38135,1732001650813'
2024-11-19T07:34:11,842 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,38135,1732001650813
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,38135,1732001650813'
2024-11-19T07:34:11,843 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:34:11,844 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:34:11,844 DEBUG [RS:0;db0afee3eab9:38135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:34:11,844 INFO  [RS:0;db0afee3eab9:38135 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:34:11,845 INFO  [RS:0;db0afee3eab9:38135 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:34:11,884 WARN  [db0afee3eab9:41723 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:34:11,947 INFO  [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C38135%2C1732001650813, suffix=, logDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/WALs/db0afee3eab9,38135,1732001650813, archiveDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/oldWALs, maxLogs=32
2024-11-19T07:34:11,951 INFO  [RS:0;db0afee3eab9:38135 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38135%2C1732001650813.1732001651951
2024-11-19T07:34:11,960 INFO  [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/WALs/db0afee3eab9,38135,1732001650813/db0afee3eab9%2C38135%2C1732001650813.1732001651951
2024-11-19T07:34:11,962 DEBUG [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42351:42351),(127.0.0.1/127.0.0.1:36597:36597)]
2024-11-19T07:34:12,135 DEBUG [db0afee3eab9:41723 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:34:12,136 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,38135,1732001650813
2024-11-19T07:34:12,139 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,38135,1732001650813, state=OPENING
2024-11-19T07:34:12,208 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:34:12,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:12,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:12,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:12,220 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:34:12,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:12,220 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,38135,1732001650813}]
2024-11-19T07:34:12,377 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:34:12,380 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37327, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:34:12,385 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:34:12,385 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:12,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:12,387 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C38135%2C1732001650813.meta, suffix=.meta, logDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/WALs/db0afee3eab9,38135,1732001650813, archiveDir=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/oldWALs, maxLogs=32
2024-11-19T07:34:12,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:12,390 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38135%2C1732001650813.meta.1732001652390.meta
2024-11-19T07:34:12,403 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/WALs/db0afee3eab9,38135,1732001650813/db0afee3eab9%2C38135%2C1732001650813.meta.1732001652390.meta
2024-11-19T07:34:12,407 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36597:36597),(127.0.0.1/127.0.0.1:42351:42351)]
2024-11-19T07:34:12,409 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:34:12,409 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:34:12,409 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:34:12,409 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:34:12,410 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:34:12,410 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:12,410 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:34:12,410 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:34:12,412 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:34:12,413 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:34:12,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:12,413 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:12,413 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:34:12,414 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:34:12,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:12,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:12,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:34:12,416 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:34:12,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:12,416 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:12,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:34:12,417 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:34:12,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:12,418 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:12,418 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:34:12,419 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740
2024-11-19T07:34:12,421 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740
2024-11-19T07:34:12,422 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:34:12,422 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:34:12,423 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:34:12,424 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:34:12,425 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=884284, jitterRate=0.12442564964294434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:34:12,425 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:34:12,426 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001652410Writing region info on filesystem at 1732001652410Initializing all the Stores at 1732001652411 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001652411Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001652411Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001652411Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001652411Cleaning up temporary data from old regions at 1732001652422 (+11 ms)Running coprocessor post-open hooks at 1732001652425 (+3 ms)Region opened successfully at 1732001652426 (+1 ms)
2024-11-19T07:34:12,427 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001652377
2024-11-19T07:34:12,431 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:34:12,431 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:34:12,432 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,38135,1732001650813
2024-11-19T07:34:12,433 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,38135,1732001650813, state=OPEN
2024-11-19T07:34:12,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:34:12,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:34:12,474 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,38135,1732001650813
2024-11-19T07:34:12,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:12,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:12,480 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:34:12,480 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,38135,1732001650813 in 254 msec
2024-11-19T07:34:12,485 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:34:12,485 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 749 msec
2024-11-19T07:34:12,487 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:12,487 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:34:12,490 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:34:12,490 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,38135,1732001650813, seqNum=-1]
2024-11-19T07:34:12,490 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:34:12,492 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48085, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:34:12,504 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 825 msec
2024-11-19T07:34:12,504 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001652504, completionTime=-1
2024-11-19T07:34:12,504 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:34:12,504 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:34:12,506 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001712507
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001772507
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:41723, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,507 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,508 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,510 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.616sec
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:34:12,513 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:34:12,516 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:34:12,516 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:34:12,516 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41723,1732001650635-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:12,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57927b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:12,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,41723,-1 for getting cluster id
2024-11-19T07:34:12,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:34:12,551 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b0f07a6a-fa8d-438c-8c95-f3c051e9e44d'
2024-11-19T07:34:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:34:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b0f07a6a-fa8d-438c-8c95-f3c051e9e44d"

2024-11-19T07:34:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fba55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,41723,-1]
2024-11-19T07:34:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:34:12,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,555 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:34:12,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa3a1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:12,557 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:34:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,38135,1732001650813, seqNum=-1]
2024-11-19T07:34:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:34:12,561 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:34:12,564 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,41723,1732001650635
2024-11-19T07:34:12,564 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:12,567 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:34:12,568 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:34:12,568 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:12,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:12,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,568 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:34:12,568 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:34:12,568 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=153670529, stopped=false
2024-11-19T07:34:12,568 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,41723,1732001650635
2024-11-19T07:34:12,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:12,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:12,590 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:34:12,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:12,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:12,591 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:12,591 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:12,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:12,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:12,591 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,38135,1732001650813' *****
2024-11-19T07:34:12,591 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:34:12,591 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:34:12,592 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,38135,1732001650813
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:38135.
2024-11-19T07:34:12,592 DEBUG [RS:0;db0afee3eab9:38135 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:12,592 DEBUG [RS:0;db0afee3eab9:38135 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:34:12,592 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:34:12,593 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T07:34:12,593 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-19T07:34:12,593 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:34:12,593 DEBUG [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T07:34:12,593 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:34:12,593 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:34:12,593 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:34:12,593 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:34:12,593 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-19T07:34:12,609 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/.tmp/ns/dd32ef6852f5435abe930779138a469b is 43, key is default/ns:d/1732001652493/Put/seqid=0
2024-11-19T07:34:12,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741835_1011 (size=5153)
2024-11-19T07:34:12,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741835_1011 (size=5153)
2024-11-19T07:34:12,615 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/.tmp/ns/dd32ef6852f5435abe930779138a469b
2024-11-19T07:34:12,623 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/.tmp/ns/dd32ef6852f5435abe930779138a469b as hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/ns/dd32ef6852f5435abe930779138a469b
2024-11-19T07:34:12,631 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/ns/dd32ef6852f5435abe930779138a469b, entries=2, sequenceid=6, filesize=5.0 K
2024-11-19T07:34:12,633 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=6, compaction requested=false
2024-11-19T07:34:12,633 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:34:12,639 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-19T07:34:12,639 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:34:12,640 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:12,640 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001652593Running coprocessor pre-close hooks at 1732001652593Disabling compacts and flushes for region at 1732001652593Disabling writes for close at 1732001652593Obtaining lock to block concurrent updates at 1732001652593Preparing flush snapshotting stores in 1588230740 at 1732001652593Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732001652594 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732001652594Flushing 1588230740/ns: creating writer at 1732001652594Flushing 1588230740/ns: appending metadata at 1732001652609 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732001652609Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3046f095: reopening flushed file at 1732001652622 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 40ms, sequenceid=6, compaction requested=false at 1732001652633 (+11 ms)Writing region close event to WAL at 1732001652635 (+2 ms)Running coprocessor post-close hooks at 1732001652639 (+4 ms)Closed at 1732001652640 (+1 ms)
2024-11-19T07:34:12,640 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:12,793 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,38135,1732001650813; all regions closed.
2024-11-19T07:34:12,794 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,795 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,795 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,795 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,795 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741834_1010 (size=1152)
2024-11-19T07:34:12,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741834_1010 (size=1152)
2024-11-19T07:34:12,804 DEBUG [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/oldWALs
2024-11-19T07:34:12,804 INFO  [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C38135%2C1732001650813.meta:.meta(num 1732001652390)
2024-11-19T07:34:12,805 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,805 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,805 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,806 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,806 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:12,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741833_1009 (size=93)
2024-11-19T07:34:12,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741833_1009 (size=93)
2024-11-19T07:34:12,812 DEBUG [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/oldWALs
2024-11-19T07:34:12,812 INFO  [RS:0;db0afee3eab9:38135 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C38135%2C1732001650813:(num 1732001651951)
2024-11-19T07:34:12,812 DEBUG [RS:0;db0afee3eab9:38135 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:12,812 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:12,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:34:12,812 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:34:12,813 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:34:12,813 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:34:12,813 INFO  [RS:0;db0afee3eab9:38135 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38135
2024-11-19T07:34:12,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,38135,1732001650813
2024-11-19T07:34:12,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:12,822 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:34:12,832 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,38135,1732001650813]
2024-11-19T07:34:12,843 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,38135,1732001650813 already deleted, retry=false
2024-11-19T07:34:12,843 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,38135,1732001650813 expired; onlineServers=0
2024-11-19T07:34:12,843 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,41723,1732001650635' *****
2024-11-19T07:34:12,843 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:34:12,843 INFO  [M:0;db0afee3eab9:41723 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:12,843 INFO  [M:0;db0afee3eab9:41723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:34:12,844 DEBUG [M:0;db0afee3eab9:41723 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:34:12,844 DEBUG [M:0;db0afee3eab9:41723 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:34:12,844 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:34:12,844 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001651684 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001651684,5,FailOnTimeoutGroup]
2024-11-19T07:34:12,844 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001651684 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001651684,5,FailOnTimeoutGroup]
2024-11-19T07:34:12,844 INFO  [M:0;db0afee3eab9:41723 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:34:12,845 INFO  [M:0;db0afee3eab9:41723 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:34:12,845 DEBUG [M:0;db0afee3eab9:41723 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:34:12,845 INFO  [M:0;db0afee3eab9:41723 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:34:12,845 INFO  [M:0;db0afee3eab9:41723 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:34:12,845 INFO  [M:0;db0afee3eab9:41723 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:34:12,846 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:34:12,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:34:12,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:12,854 DEBUG [M:0;db0afee3eab9:41723 {}] zookeeper.ZKUtil(347): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:34:12,854 WARN  [M:0;db0afee3eab9:41723 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:34:12,855 INFO  [M:0;db0afee3eab9:41723 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/.lastflushedseqids
2024-11-19T07:34:12,866 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741836_1012 (size=99)
2024-11-19T07:34:12,866 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741836_1012 (size=99)
2024-11-19T07:34:12,866 INFO  [M:0;db0afee3eab9:41723 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:34:12,866 INFO  [M:0;db0afee3eab9:41723 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:34:12,867 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:34:12,867 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:12,867 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:12,867 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:34:12,867 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:12,867 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-19T07:34:12,888 DEBUG [M:0;db0afee3eab9:41723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e7d9b9185e41d6bc0dd7032c2c499c is 82, key is hbase:meta,,1/info:regioninfo/1732001652432/Put/seqid=0
2024-11-19T07:34:12,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741837_1013 (size=5672)
2024-11-19T07:34:12,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741837_1013 (size=5672)
2024-11-19T07:34:12,894 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e7d9b9185e41d6bc0dd7032c2c499c
2024-11-19T07:34:12,902 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:34:12,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:12,915 DEBUG [M:0;db0afee3eab9:41723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b1d0862af1245828232b8b69f0b710a is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732001652502/Put/seqid=0
2024-11-19T07:34:12,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741838_1014 (size=5275)
2024-11-19T07:34:12,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741838_1014 (size=5275)
2024-11-19T07:34:12,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:12,923 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b1d0862af1245828232b8b69f0b710a
2024-11-19T07:34:12,932 INFO  [RS:0;db0afee3eab9:38135 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:34:12,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:12,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38135-0x1015203d3650001, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:12,932 INFO  [RS:0;db0afee3eab9:38135 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,38135,1732001650813; zookeeper connection closed.
2024-11-19T07:34:12,933 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21bc12e9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21bc12e9
2024-11-19T07:34:12,933 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:34:12,952 DEBUG [M:0;db0afee3eab9:41723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c5ea73b65f549bbbc973b4e8a33a727 is 69, key is db0afee3eab9,38135,1732001650813/rs:state/1732001651785/Put/seqid=0
2024-11-19T07:34:12,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741839_1015 (size=5156)
2024-11-19T07:34:12,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741839_1015 (size=5156)
2024-11-19T07:34:12,958 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c5ea73b65f549bbbc973b4e8a33a727
2024-11-19T07:34:12,984 DEBUG [M:0;db0afee3eab9:41723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da92d610e570459a976ce51430456809 is 52, key is load_balancer_on/state:d/1732001652566/Put/seqid=0
2024-11-19T07:34:12,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741840_1016 (size=5056)
2024-11-19T07:34:12,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741840_1016 (size=5056)
2024-11-19T07:34:12,990 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da92d610e570459a976ce51430456809
2024-11-19T07:34:12,997 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e7d9b9185e41d6bc0dd7032c2c499c as hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0e7d9b9185e41d6bc0dd7032c2c499c
2024-11-19T07:34:13,004 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0e7d9b9185e41d6bc0dd7032c2c499c, entries=8, sequenceid=29, filesize=5.5 K
2024-11-19T07:34:13,006 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b1d0862af1245828232b8b69f0b710a as hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3b1d0862af1245828232b8b69f0b710a
2024-11-19T07:34:13,013 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3b1d0862af1245828232b8b69f0b710a, entries=3, sequenceid=29, filesize=5.2 K
2024-11-19T07:34:13,014 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6c5ea73b65f549bbbc973b4e8a33a727 as hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6c5ea73b65f549bbbc973b4e8a33a727
2024-11-19T07:34:13,020 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6c5ea73b65f549bbbc973b4e8a33a727, entries=1, sequenceid=29, filesize=5.0 K
2024-11-19T07:34:13,022 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da92d610e570459a976ce51430456809 as hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/da92d610e570459a976ce51430456809
2024-11-19T07:34:13,029 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42993/user/jenkins/test-data/7427e797-b756-0d97-84a7-161792d29807/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/da92d610e570459a976ce51430456809, entries=1, sequenceid=29, filesize=4.9 K
2024-11-19T07:34:13,031 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=29, compaction requested=false
2024-11-19T07:34:13,032 INFO  [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:13,032 DEBUG [M:0;db0afee3eab9:41723 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001652867Disabling compacts and flushes for region at 1732001652867Disabling writes for close at 1732001652867Obtaining lock to block concurrent updates at 1732001652867Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001652867Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732001652868 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001652869 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001652869Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001652888 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001652888Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001652900 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001652915 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001652915Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001652930 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001652951 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001652951Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001652963 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001652983 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001652983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c2d0401: reopening flushed file at 1732001652996 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a907641: reopening flushed file at 1732001653004 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56bbe690: reopening flushed file at 1732001653013 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@410498f0: reopening flushed file at 1732001653021 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=29, compaction requested=false at 1732001653031 (+10 ms)Writing region close event to WAL at 1732001653032 (+1 ms)Closed at 1732001653032
2024-11-19T07:34:13,033 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:13,033 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:13,033 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:13,033 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:13,033 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:13,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34021 is added to blk_1073741830_1006 (size=10311)
2024-11-19T07:34:13,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38797 is added to blk_1073741830_1006 (size=10311)
2024-11-19T07:34:13,037 INFO  [M:0;db0afee3eab9:41723 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:34:13,037 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:34:13,037 INFO  [M:0;db0afee3eab9:41723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41723
2024-11-19T07:34:13,037 INFO  [M:0;db0afee3eab9:41723 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:34:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:13,164 INFO  [M:0;db0afee3eab9:41723 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:34:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41723-0x1015203d3650000, quorum=127.0.0.1:50279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:13,168 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f5c23ef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:13,168 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@716c7b87{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:13,168 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:13,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc8c098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:13,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b915b67{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:13,170 WARN  [BP-1712531913-172.17.0.2-1732001648026 heartbeating to localhost/127.0.0.1:42993 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:13,170 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:13,170 WARN  [BP-1712531913-172.17.0.2-1732001648026 heartbeating to localhost/127.0.0.1:42993 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1712531913-172.17.0.2-1732001648026 (Datanode Uuid 29f38b1b-e788-4823-84ac-21efa4e05822) service to localhost/127.0.0.1:42993
2024-11-19T07:34:13,170 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:13,171 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data3/current/BP-1712531913-172.17.0.2-1732001648026 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:13,172 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data4/current/BP-1712531913-172.17.0.2-1732001648026 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:13,172 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:13,175 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@52b07bdb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:13,175 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70121b28{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:13,175 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:13,176 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aa9354f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:13,176 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c8914e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:13,178 WARN  [BP-1712531913-172.17.0.2-1732001648026 heartbeating to localhost/127.0.0.1:42993 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:13,178 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:13,178 WARN  [BP-1712531913-172.17.0.2-1732001648026 heartbeating to localhost/127.0.0.1:42993 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1712531913-172.17.0.2-1732001648026 (Datanode Uuid 792ddadc-f6f2-45e3-84a8-7132247f4c12) service to localhost/127.0.0.1:42993
2024-11-19T07:34:13,178 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:13,179 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data1/current/BP-1712531913-172.17.0.2-1732001648026 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:13,179 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/cluster_3ea1d3f9-13dc-a85c-e800-4e255835ec39/data/data2/current/BP-1712531913-172.17.0.2-1732001648026 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:13,179 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:13,185 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@493d1d34{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:34:13,185 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6a249094{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:13,185 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:13,186 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75cbfab9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:13,186 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65506a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:13,191 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.log.dir so I do NOT create it in target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9aabb9e6-67aa-3814-8527-1fc9888259cd/hadoop.tmp.dir so I do NOT create it in target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7, deleteOnExit=true
2024-11-19T07:34:13,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/test.cache.data in system properties and HBase conf
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:34:13,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:34:13,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:34:13,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:34:13,221 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:34:13,719 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:13,727 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:13,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:13,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:13,730 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:13,733 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:13,735 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:13,735 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:13,814 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:13,830 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2606b08f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-45121-hadoop-hdfs-3_4_1-tests_jar-_-any-2440794384290381465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:34:13,830 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:45121}
2024-11-19T07:34:13,830 INFO  [Time-limited test {}] server.Server(415): Started @110519ms
2024-11-19T07:34:13,843 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:34:14,104 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:14,108 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:14,112 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:14,112 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:14,112 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:14,113 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2cb9bebc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:14,114 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf32f74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:14,210 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c77eea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-39835-hadoop-hdfs-3_4_1-tests_jar-_-any-12210601010710888158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:14,210 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e20426d{HTTP/1.1, (http/1.1)}{localhost:39835}
2024-11-19T07:34:14,210 INFO  [Time-limited test {}] server.Server(415): Started @110899ms
2024-11-19T07:34:14,212 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:14,239 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:14,242 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:14,243 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:14,243 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:14,243 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:34:14,243 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f7f19bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:14,244 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32403ac6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:14,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6cd7b3e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-38977-hadoop-hdfs-3_4_1-tests_jar-_-any-3519644253316810858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:14,343 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ff95875{HTTP/1.1, (http/1.1)}{localhost:38977}
2024-11-19T07:34:14,343 INFO  [Time-limited test {}] server.Server(415): Started @111032ms
2024-11-19T07:34:14,344 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:15,790 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data1/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:15,790 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data2/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:15,808 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:15,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f0fbbd969861ba9 with lease ID 0x83a199dfabd8ed09: Processing first storage report for DS-66c39e22-4de6-4e40-8632-497e67c5d05c from datanode DatanodeRegistration(127.0.0.1:44027, datanodeUuid=93814c7a-5f63-4340-a4d2-45d183aecc07, infoPort=32769, infoSecurePort=0, ipcPort=39579, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:15,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f0fbbd969861ba9 with lease ID 0x83a199dfabd8ed09: from storage DS-66c39e22-4de6-4e40-8632-497e67c5d05c node DatanodeRegistration(127.0.0.1:44027, datanodeUuid=93814c7a-5f63-4340-a4d2-45d183aecc07, infoPort=32769, infoSecurePort=0, ipcPort=39579, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:15,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f0fbbd969861ba9 with lease ID 0x83a199dfabd8ed09: Processing first storage report for DS-dc6c2ec0-810f-4dc5-93c4-ae60af3a8dc6 from datanode DatanodeRegistration(127.0.0.1:44027, datanodeUuid=93814c7a-5f63-4340-a4d2-45d183aecc07, infoPort=32769, infoSecurePort=0, ipcPort=39579, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:15,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f0fbbd969861ba9 with lease ID 0x83a199dfabd8ed09: from storage DS-dc6c2ec0-810f-4dc5-93c4-ae60af3a8dc6 node DatanodeRegistration(127.0.0.1:44027, datanodeUuid=93814c7a-5f63-4340-a4d2-45d183aecc07, infoPort=32769, infoSecurePort=0, ipcPort=39579, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:15,928 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data3/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:15,928 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data4/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:15,953 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:15,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x478cb5160f762cca with lease ID 0x83a199dfabd8ed0a: Processing first storage report for DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3 from datanode DatanodeRegistration(127.0.0.1:35925, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=45869, infoSecurePort=0, ipcPort=43839, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:15,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x478cb5160f762cca with lease ID 0x83a199dfabd8ed0a: from storage DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3 node DatanodeRegistration(127.0.0.1:35925, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=45869, infoSecurePort=0, ipcPort=43839, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:15,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x478cb5160f762cca with lease ID 0x83a199dfabd8ed0a: Processing first storage report for DS-f1c1ee06-b282-494f-bbf3-95152a77ee1e from datanode DatanodeRegistration(127.0.0.1:35925, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=45869, infoSecurePort=0, ipcPort=43839, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:15,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x478cb5160f762cca with lease ID 0x83a199dfabd8ed0a: from storage DS-f1c1ee06-b282-494f-bbf3-95152a77ee1e node DatanodeRegistration(127.0.0.1:35925, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=45869, infoSecurePort=0, ipcPort=43839, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:15,984 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6
2024-11-19T07:34:15,987 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/zookeeper_0, clientPort=62643, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:34:15,988 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62643
2024-11-19T07:34:15,988 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:15,990 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:34:16,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:34:16,002 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332 with version=8
2024-11-19T07:34:16,002 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:34:16,005 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:34:16,005 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:34:16,006 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:34:16,007 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42985
2024-11-19T07:34:16,009 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42985 connecting to ZooKeeper ensemble=127.0.0.1:62643
2024-11-19T07:34:16,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429850x0, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:34:16,064 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42985-0x1015203e85d0000 connected
2024-11-19T07:34:16,146 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,148 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:16,151 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332, hbase.cluster.distributed=false
2024-11-19T07:34:16,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:34:16,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42985
2024-11-19T07:34:16,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42985
2024-11-19T07:34:16,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42985
2024-11-19T07:34:16,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42985
2024-11-19T07:34:16,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42985
2024-11-19T07:34:16,169 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:34:16,169 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:34:16,170 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:34:16,171 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42239
2024-11-19T07:34:16,172 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42239 connecting to ZooKeeper ensemble=127.0.0.1:62643
2024-11-19T07:34:16,173 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,175 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422390x0, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:34:16,189 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:16,189 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42239-0x1015203e85d0001 connected
2024-11-19T07:34:16,189 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:34:16,190 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:34:16,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:34:16,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:34:16,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42239
2024-11-19T07:34:16,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42239
2024-11-19T07:34:16,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42239
2024-11-19T07:34:16,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42239
2024-11-19T07:34:16,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42239
2024-11-19T07:34:16,206 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:42985
2024-11-19T07:34:16,207 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:16,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:16,220 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:34:16,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,231 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:34:16,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,42985,1732001656005 from backup master directory
2024-11-19T07:34:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:16,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:34:16,241 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:34:16,241 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,246 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/hbase.id] with ID: e2f053c6-67cc-4f4e-b98e-d8c039030724
2024-11-19T07:34:16,247 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/.tmp/hbase.id
2024-11-19T07:34:16,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:34:16,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:34:16,254 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/.tmp/hbase.id]:[hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/hbase.id]
2024-11-19T07:34:16,269 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:16,269 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:34:16,271 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T07:34:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:34:16,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:34:16,290 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:34:16,291 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:34:16,291 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:16,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:34:16,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:34:16,302 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store
2024-11-19T07:34:16,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:34:16,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:34:16,309 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:16,310 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:34:16,310 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:16,310 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:16,310 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:34:16,310 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:16,310 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:34:16,310 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001656309Disabling compacts and flushes for region at 1732001656309Disabling writes for close at 1732001656310 (+1 ms)Writing region close event to WAL at 1732001656310Closed at 1732001656310
2024-11-19T07:34:16,311 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/.initializing
2024-11-19T07:34:16,311 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,313 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42985%2C1732001656005, suffix=, logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005, archiveDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/oldWALs, maxLogs=10
2024-11-19T07:34:16,314 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42985%2C1732001656005.1732001656313
2024-11-19T07:34:16,319 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313
2024-11-19T07:34:16,320 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45869:45869),(127.0.0.1/127.0.0.1:32769:32769)]
2024-11-19T07:34:16,322 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:34:16,322 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:16,322 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,322 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,324 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:16,325 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:34:16,326 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,326 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:16,326 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:16,328 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:34:16,328 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,328 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:16,328 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:16,330 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:34:16,330 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,330 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:16,330 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:34:16,332 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:34:16,332 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,333 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:16,333 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,333 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,334 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,335 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,335 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,336 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:34:16,337 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:34:16,340 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:34:16,341 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756173, jitterRate=-0.038477301597595215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:34:16,342 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001656322Initializing all the Stores at 1732001656323 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001656323Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001656324 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001656324Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001656324Cleaning up temporary data from old regions at 1732001656335 (+11 ms)Region opened successfully at 1732001656342 (+7 ms)
2024-11-19T07:34:16,342 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:34:16,347 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73766fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:34:16,348 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:34:16,348 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:34:16,348 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:34:16,348 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:34:16,349 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:34:16,350 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:34:16,350 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:34:16,352 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:34:16,354 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:34:16,356 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:34:16,357 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:34:16,357 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:34:16,367 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:34:16,367 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:34:16,368 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:34:16,377 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:34:16,379 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:34:16,388 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:34:16,392 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:34:16,399 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:34:16,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:16,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:16,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,410 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,42985,1732001656005, sessionid=0x1015203e85d0000, setting cluster-up flag (Was=false)
2024-11-19T07:34:16,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,462 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:34:16,464 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,515 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:34:16,517 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,42985,1732001656005
2024-11-19T07:34:16,518 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:34:16,521 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:16,521 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:34:16,521 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:34:16,522 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,42985,1732001656005
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:34:16,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,525 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001686525
2024-11-19T07:34:16,525 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,526 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:16,526 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:34:16,526 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:34:16,527 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:34:16,527 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:34:16,527 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:34:16,527 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:34:16,528 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,528 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:34:16,531 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001656527,5,FailOnTimeoutGroup]
2024-11-19T07:34:16,532 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001656531,5,FailOnTimeoutGroup]
2024-11-19T07:34:16,532 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,532 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:34:16,532 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,532 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:34:16,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:34:16,539 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:34:16,539 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332
2024-11-19T07:34:16,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:34:16,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:34:16,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:16,548 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:34:16,549 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:34:16,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,550 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:16,550 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:34:16,552 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:34:16,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,552 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:16,552 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:34:16,554 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:34:16,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,554 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:16,554 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:34:16,556 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:34:16,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:16,556 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:16,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:34:16,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740
2024-11-19T07:34:16,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740
2024-11-19T07:34:16,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:34:16,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:34:16,559 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:34:16,560 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:34:16,562 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:34:16,563 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860704, jitterRate=0.09444290399551392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:34:16,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001656547Initializing all the Stores at 1732001656547Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001656548 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001656548Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001656548Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001656548Cleaning up temporary data from old regions at 1732001656558 (+10 ms)Region opened successfully at 1732001656564 (+6 ms)
2024-11-19T07:34:16,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:34:16,564 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:34:16,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:34:16,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:34:16,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:34:16,565 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:16,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001656564Disabling compacts and flushes for region at 1732001656564Disabling writes for close at 1732001656564Writing region close event to WAL at 1732001656565 (+1 ms)Closed at 1732001656565
2024-11-19T07:34:16,566 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:16,566 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:34:16,566 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:34:16,568 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:34:16,569 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:34:16,598 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(746): ClusterId : e2f053c6-67cc-4f4e-b98e-d8c039030724
2024-11-19T07:34:16,598 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:34:16,611 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:34:16,611 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:34:16,621 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:34:16,621 DEBUG [RS:0;db0afee3eab9:42239 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b992c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:34:16,633 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:42239
2024-11-19T07:34:16,633 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:34:16,633 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:34:16,633 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:34:16,634 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,42985,1732001656005 with port=42239, startcode=1732001656169
2024-11-19T07:34:16,634 DEBUG [RS:0;db0afee3eab9:42239 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:34:16,636 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:34:16,637 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42985 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,637 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42985 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,639 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332
2024-11-19T07:34:16,639 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40659
2024-11-19T07:34:16,639 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:34:16,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:16,652 DEBUG [RS:0;db0afee3eab9:42239 {}] zookeeper.ZKUtil(111): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,652 WARN  [RS:0;db0afee3eab9:42239 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:34:16,652 INFO  [RS:0;db0afee3eab9:42239 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:16,652 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,652 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,42239,1732001656169]
2024-11-19T07:34:16,657 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:34:16,661 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:34:16,661 INFO  [RS:0;db0afee3eab9:42239 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:34:16,661 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,662 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:34:16,664 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:34:16,664 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,664 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,664 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,665 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,666 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,666 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:16,666 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:16,666 DEBUG [RS:0;db0afee3eab9:42239 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:16,666 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,667 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,667 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,667 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,667 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,667 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42239,1732001656169-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:34:16,681 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:34:16,681 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42239,1732001656169-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,681 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,682 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.Replication(171): db0afee3eab9,42239,1732001656169 started
2024-11-19T07:34:16,695 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:16,696 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,42239,1732001656169, RpcServer on db0afee3eab9/172.17.0.2:42239, sessionid=0x1015203e85d0001
2024-11-19T07:34:16,696 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:34:16,696 DEBUG [RS:0;db0afee3eab9:42239 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,696 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42239,1732001656169'
2024-11-19T07:34:16,696 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42239,1732001656169'
2024-11-19T07:34:16,697 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:34:16,698 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:34:16,698 DEBUG [RS:0;db0afee3eab9:42239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:34:16,698 INFO  [RS:0;db0afee3eab9:42239 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:34:16,698 INFO  [RS:0;db0afee3eab9:42239 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:34:16,719 WARN  [db0afee3eab9:42985 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:34:16,801 INFO  [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42239%2C1732001656169, suffix=, logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169, archiveDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs, maxLogs=32
2024-11-19T07:34:16,802 INFO  [RS:0;db0afee3eab9:42239 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42239%2C1732001656169.1732001656802
2024-11-19T07:34:16,831 INFO  [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
2024-11-19T07:34:16,832 DEBUG [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45869:45869),(127.0.0.1/127.0.0.1:32769:32769)]
2024-11-19T07:34:16,970 DEBUG [db0afee3eab9:42985 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:34:16,970 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,42239,1732001656169
2024-11-19T07:34:16,972 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42239,1732001656169, state=OPENING
2024-11-19T07:34:16,977 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:34:16,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:16,989 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:34:16,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:16,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:16,990 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42239,1732001656169}]
2024-11-19T07:34:17,146 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:34:17,151 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54397, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:34:17,157 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:34:17,158 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:17,162 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42239%2C1732001656169.meta, suffix=.meta, logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169, archiveDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs, maxLogs=32
2024-11-19T07:34:17,163 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
2024-11-19T07:34:17,170 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
2024-11-19T07:34:17,173 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45869:45869),(127.0.0.1/127.0.0.1:32769:32769)]
2024-11-19T07:34:17,174 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:34:17,175 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:34:17,175 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:34:17,177 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:34:17,178 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:34:17,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:17,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:17,179 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:34:17,179 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:34:17,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:17,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:17,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:34:17,181 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:34:17,181 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:17,181 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:17,181 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:34:17,182 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:34:17,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:17,182 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:34:17,183 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:34:17,183 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740
2024-11-19T07:34:17,184 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740
2024-11-19T07:34:17,186 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:34:17,186 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:34:17,186 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:34:17,188 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:34:17,189 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703622, jitterRate=-0.10529907047748566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:34:17,189 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:34:17,189 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001657176Writing region info on filesystem at 1732001657176Initializing all the Stores at 1732001657177 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001657177Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001657177Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001657177Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001657177Cleaning up temporary data from old regions at 1732001657186 (+9 ms)Running coprocessor post-open hooks at 1732001657189 (+3 ms)Region opened successfully at 1732001657189
2024-11-19T07:34:17,190 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001657146
2024-11-19T07:34:17,193 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:34:17,193 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:34:17,194 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42239,1732001656169
2024-11-19T07:34:17,195 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42239,1732001656169, state=OPEN
2024-11-19T07:34:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:34:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:34:17,307 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,42239,1732001656169
2024-11-19T07:34:17,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:17,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:34:17,311 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:34:17,311 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42239,1732001656169 in 317 msec
2024-11-19T07:34:17,314 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:34:17,314 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 745 msec
2024-11-19T07:34:17,315 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:34:17,315 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:34:17,316 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:34:17,317 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42239,1732001656169, seqNum=-1]
2024-11-19T07:34:17,317 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:34:17,318 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:34:17,325 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 803 msec
2024-11-19T07:34:17,325 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001657325, completionTime=-1
2024-11-19T07:34:17,325 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:34:17,325 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:34:17,328 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:34:17,328 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001717328
2024-11-19T07:34:17,328 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001777328
2024-11-19T07:34:17,328 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:42985, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,329 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,331 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:34:17,334 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.093sec
2024-11-19T07:34:17,334 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:34:17,334 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:34:17,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:34:17,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:34:17,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:34:17,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:34:17,335 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:34:17,338 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:34:17,338 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:34:17,338 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42985,1732001656005-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7982f2bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:17,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,42985,-1 for getting cluster id
2024-11-19T07:34:17,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:34:17,401 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e2f053c6-67cc-4f4e-b98e-d8c039030724'
2024-11-19T07:34:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:34:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e2f053c6-67cc-4f4e-b98e-d8c039030724"

2024-11-19T07:34:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff0542e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,42985,-1]
2024-11-19T07:34:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:34:17,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:17,405 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:34:17,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be29fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:34:17,407 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:34:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42239,1732001656169, seqNum=-1]
2024-11-19T07:34:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:34:17,410 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:34:17,413 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,42985,1732001656005
2024-11-19T07:34:17,414 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:17,417 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:34:17,436 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:34:17,436 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:17,436 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:17,436 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:34:17,436 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:34:17,437 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:34:17,437 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:34:17,437 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:34:17,438 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37853
2024-11-19T07:34:17,440 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37853 connecting to ZooKeeper ensemble=127.0.0.1:62643
2024-11-19T07:34:17,441 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:17,442 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:34:17,462 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378530x0, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:34:17,462 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37853-0x1015203e85d0002 connected
2024-11-19T07:34:17,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-19T07:34:17,462 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-19T07:34:17,463 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:34:17,464 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:34:17,464 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:34:17,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:34:17,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37853
2024-11-19T07:34:17,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37853
2024-11-19T07:34:17,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37853
2024-11-19T07:34:17,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37853
2024-11-19T07:34:17,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37853
2024-11-19T07:34:17,475 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(746): ClusterId : e2f053c6-67cc-4f4e-b98e-d8c039030724
2024-11-19T07:34:17,475 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:34:17,484 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:34:17,484 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:34:17,494 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:34:17,495 DEBUG [RS:1;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba5bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:34:17,508 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;db0afee3eab9:37853
2024-11-19T07:34:17,508 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:34:17,508 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:34:17,508 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:34:17,509 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,42985,1732001656005 with port=37853, startcode=1732001657436
2024-11-19T07:34:17,509 DEBUG [RS:1;db0afee3eab9:37853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:34:17,510 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:34:17,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42985 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42985 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,512 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332
2024-11-19T07:34:17,512 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40659
2024-11-19T07:34:17,512 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:34:17,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:17,525 DEBUG [RS:1;db0afee3eab9:37853 {}] zookeeper.ZKUtil(111): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,526 WARN  [RS:1;db0afee3eab9:37853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:34:17,526 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,37853,1732001657436]
2024-11-19T07:34:17,526 INFO  [RS:1;db0afee3eab9:37853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:34:17,526 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,531 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:34:17,533 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:34:17,534 INFO  [RS:1;db0afee3eab9:37853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:34:17,534 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,534 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:34:17,535 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:34:17,536 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,536 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,537 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,537 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,537 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:34:17,537 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:17,537 DEBUG [RS:1;db0afee3eab9:37853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:34:17,537 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,537 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,537 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,537 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,538 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,538 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37853,1732001657436-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:34:17,551 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:34:17,551 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37853,1732001657436-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,551 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,552 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.Replication(171): db0afee3eab9,37853,1732001657436 started
2024-11-19T07:34:17,564 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:34:17,564 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,37853,1732001657436, RpcServer on db0afee3eab9/172.17.0.2:37853, sessionid=0x1015203e85d0002
2024-11-19T07:34:17,564 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:34:17,564 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;db0afee3eab9:37853,5,FailOnTimeoutGroup]
2024-11-19T07:34:17,564 DEBUG [RS:1;db0afee3eab9:37853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,564 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37853,1732001657436'
2024-11-19T07:34:17,564 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:34:17,564 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-19T07:34:17,565 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T07:34:17,565 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:34:17,565 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:34:17,565 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:34:17,565 DEBUG [RS:1;db0afee3eab9:37853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,37853,1732001657436
2024-11-19T07:34:17,565 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37853,1732001657436'
2024-11-19T07:34:17,566 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:34:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,42985,1732001656005
2024-11-19T07:34:17,566 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:34:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6c57f553
2024-11-19T07:34:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T07:34:17,566 DEBUG [RS:1;db0afee3eab9:37853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:34:17,566 INFO  [RS:1;db0afee3eab9:37853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:34:17,566 INFO  [RS:1;db0afee3eab9:37853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:34:17,568 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T07:34:17,568 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T07:34:17,568 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T07:34:17,569 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:34:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T07:34:17,571 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T07:34:17,571 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:17,571 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-19T07:34:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:34:17,573 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T07:34:17,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741835_1011 (size=393)
2024-11-19T07:34:17,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741835_1011 (size=393)
2024-11-19T07:34:17,580 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1336ae17c9e49afba8075a95cc1255a1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332
2024-11-19T07:34:17,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44027 is added to blk_1073741836_1012 (size=76)
2024-11-19T07:34:17,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35925 is added to blk_1073741836_1012 (size=76)
2024-11-19T07:34:17,672 INFO  [RS:1;db0afee3eab9:37853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37853%2C1732001657436, suffix=, logDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436, archiveDir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs, maxLogs=32
2024-11-19T07:34:17,673 INFO  [RS:1;db0afee3eab9:37853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001657673
2024-11-19T07:34:17,682 INFO  [RS:1;db0afee3eab9:37853 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673
2024-11-19T07:34:17,683 DEBUG [RS:1;db0afee3eab9:37853 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32769:32769),(127.0.0.1/127.0.0.1:45869:45869)]
2024-11-19T07:34:17,913 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:34:17,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:17,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:17,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:17,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:17,990 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:17,990 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 1336ae17c9e49afba8075a95cc1255a1, disabling compactions & flushes
2024-11-19T07:34:17,990 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:17,990 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:17,990 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1. after waiting 0 ms
2024-11-19T07:34:17,990 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:17,990 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:17,991 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1336ae17c9e49afba8075a95cc1255a1:
Waiting for close lock at 1732001657990Disabling compacts and flushes for region at 1732001657990Disabling writes for close at 1732001657990Writing region close event to WAL at 1732001657990Closed at 1732001657990
2024-11-19T07:34:17,993 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T07:34:17,994 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732001657993"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001657993"}]},"ts":"1732001657993"}
2024-11-19T07:34:17,997 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T07:34:17,999 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T07:34:18,000 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001657999"}]},"ts":"1732001657999"}
2024-11-19T07:34:18,003 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-19T07:34:18,006 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {db0afee3eab9=0} racks are {/default-rack=0}
2024-11-19T07:34:18,010 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-19T07:34:18,010 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-19T07:34:18,010 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-19T07:34:18,010 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-19T07:34:18,010 INFO  [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-19T07:34:18,010 INFO  [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-19T07:34:18,010 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-19T07:34:18,011 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1336ae17c9e49afba8075a95cc1255a1, ASSIGN}]
2024-11-19T07:34:18,013 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1336ae17c9e49afba8075a95cc1255a1, ASSIGN
2024-11-19T07:34:18,014 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1336ae17c9e49afba8075a95cc1255a1, ASSIGN; state=OFFLINE, location=db0afee3eab9,37853,1732001657436; forceNewPlan=false, retain=false
2024-11-19T07:34:18,165 INFO  [db0afee3eab9:42985 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-19T07:34:18,165 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1336ae17c9e49afba8075a95cc1255a1, regionState=OPENING, regionLocation=db0afee3eab9,37853,1732001657436
2024-11-19T07:34:18,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1336ae17c9e49afba8075a95cc1255a1, ASSIGN because future has completed
2024-11-19T07:34:18,168 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1336ae17c9e49afba8075a95cc1255a1, server=db0afee3eab9,37853,1732001657436}]
2024-11-19T07:34:18,322 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:34:18,327 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:34:18,335 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:18,335 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1336ae17c9e49afba8075a95cc1255a1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:34:18,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:34:18,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,336 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,338 INFO  [StoreOpener-1336ae17c9e49afba8075a95cc1255a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1336ae17c9e49afba8075a95cc1255a1 
2024-11-19T07:34:18,339 INFO  [StoreOpener-1336ae17c9e49afba8075a95cc1255a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1336ae17c9e49afba8075a95cc1255a1 columnFamilyName info
2024-11-19T07:34:18,340 DEBUG [StoreOpener-1336ae17c9e49afba8075a95cc1255a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:34:18,340 INFO  [StoreOpener-1336ae17c9e49afba8075a95cc1255a1-1 {}] regionserver.HStore(327): Store=1336ae17c9e49afba8075a95cc1255a1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:34:18,340 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,341 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,342 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,342 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,343 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,345 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,348 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:34:18,348 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1336ae17c9e49afba8075a95cc1255a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=740858, jitterRate=-0.05795152485370636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:34:18,348 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:18,350 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1336ae17c9e49afba8075a95cc1255a1:
Running coprocessor pre-open hook at 1732001658336Writing region info on filesystem at 1732001658336Initializing all the Stores at 1732001658337 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001658337Cleaning up temporary data from old regions at 1732001658343 (+6 ms)Running coprocessor post-open hooks at 1732001658348 (+5 ms)Region opened successfully at 1732001658350 (+2 ms)
2024-11-19T07:34:18,351 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., pid=6, masterSystemTime=1732001658322
2024-11-19T07:34:18,354 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:18,354 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:18,355 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1336ae17c9e49afba8075a95cc1255a1, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37853,1732001657436
2024-11-19T07:34:18,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1336ae17c9e49afba8075a95cc1255a1, server=db0afee3eab9,37853,1732001657436 because future has completed
2024-11-19T07:34:18,364 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T07:34:18,364 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1336ae17c9e49afba8075a95cc1255a1, server=db0afee3eab9,37853,1732001657436 in 192 msec
2024-11-19T07:34:18,366 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T07:34:18,367 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1336ae17c9e49afba8075a95cc1255a1, ASSIGN in 353 msec
2024-11-19T07:34:18,368 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T07:34:18,368 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001658368"}]},"ts":"1732001658368"}
2024-11-19T07:34:18,370 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-19T07:34:18,372 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T07:34:18,374 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 803 msec
2024-11-19T07:34:21,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:34:21,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T07:34:21,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T07:34:21,080 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-19T07:34:21,082 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:34:21,082 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T07:34:21,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:34:21,083 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T07:34:23,839 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:34:23,841 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:23,859 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:23,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:23,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:34:23,868 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-19T07:34:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:34:27,629 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-19T07:34:27,629 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-19T07:34:27,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T07:34:27,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:27,650 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:27,655 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:27,656 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:27,656 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:27,656 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:27,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45d50f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:27,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cac6b83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:27,772 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15751333{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-43293-hadoop-hdfs-3_4_1-tests_jar-_-any-9495811907661476646/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:27,773 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@184b7b23{HTTP/1.1, (http/1.1)}{localhost:43293}
2024-11-19T07:34:27,773 INFO  [Time-limited test {}] server.Server(415): Started @124462ms
2024-11-19T07:34:27,774 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:27,841 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:27,845 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:27,852 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:27,852 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:27,852 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:27,853 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed35b1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:27,854 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3eb2cdd3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:27,966 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49584ab0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-33513-hadoop-hdfs-3_4_1-tests_jar-_-any-7548409679409239278/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:27,967 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59d0863f{HTTP/1.1, (http/1.1)}{localhost:33513}
2024-11-19T07:34:27,967 INFO  [Time-limited test {}] server.Server(415): Started @124656ms
2024-11-19T07:34:27,968 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:28,030 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:28,035 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:28,036 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:28,036 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:28,036 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:28,037 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5455501c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:28,038 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264a677b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:28,148 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@33f262d4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-35947-hadoop-hdfs-3_4_1-tests_jar-_-any-15468386073070459677/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:28,149 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44692c68{HTTP/1.1, (http/1.1)}{localhost:35947}
2024-11-19T07:34:28,149 INFO  [Time-limited test {}] server.Server(415): Started @124837ms
2024-11-19T07:34:28,150 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:29,250 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data5/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,250 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data6/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,270 WARN  [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:29,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xceff0f21faa9015c with lease ID 0x83a199dfabd8ed0b: Processing first storage report for DS-279227df-06e6-42dd-b2a7-b11135e8d57c from datanode DatanodeRegistration(127.0.0.1:42413, datanodeUuid=66f79c7b-09c8-4374-9926-17250bcc504d, infoPort=36355, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xceff0f21faa9015c with lease ID 0x83a199dfabd8ed0b: from storage DS-279227df-06e6-42dd-b2a7-b11135e8d57c node DatanodeRegistration(127.0.0.1:42413, datanodeUuid=66f79c7b-09c8-4374-9926-17250bcc504d, infoPort=36355, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xceff0f21faa9015c with lease ID 0x83a199dfabd8ed0b: Processing first storage report for DS-f7d2eb1e-a6e4-4caf-929d-cb7297b9afc6 from datanode DatanodeRegistration(127.0.0.1:42413, datanodeUuid=66f79c7b-09c8-4374-9926-17250bcc504d, infoPort=36355, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xceff0f21faa9015c with lease ID 0x83a199dfabd8ed0b: from storage DS-f7d2eb1e-a6e4-4caf-929d-cb7297b9afc6 node DatanodeRegistration(127.0.0.1:42413, datanodeUuid=66f79c7b-09c8-4374-9926-17250bcc504d, infoPort=36355, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,611 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,611 WARN  [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,633 WARN  [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:29,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9075978cfc9661f5 with lease ID 0x83a199dfabd8ed0c: Processing first storage report for DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af from datanode DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9075978cfc9661f5 with lease ID 0x83a199dfabd8ed0c: from storage DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af node DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9075978cfc9661f5 with lease ID 0x83a199dfabd8ed0c: Processing first storage report for DS-fdb9fbf9-9ae5-428e-8c67-f5223902fad3 from datanode DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9075978cfc9661f5 with lease ID 0x83a199dfabd8ed0c: from storage DS-fdb9fbf9-9ae5-428e-8c67-f5223902fad3 node DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,728 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data9/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,728 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data10/current/BP-1194886781-172.17.0.2-1732001653232/current, will proceed with Du for space computation calculation, 
2024-11-19T07:34:29,746 WARN  [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:29,748 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ff337dfb85ec321 with lease ID 0x83a199dfabd8ed0d: Processing first storage report for DS-fb9392d4-064c-4a30-a923-0bd33a6086c4 from datanode DatanodeRegistration(127.0.0.1:38583, datanodeUuid=1e03318e-cf1d-4f81-b62e-69f76a2f240a, infoPort=38051, infoSecurePort=0, ipcPort=43547, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,749 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ff337dfb85ec321 with lease ID 0x83a199dfabd8ed0d: from storage DS-fb9392d4-064c-4a30-a923-0bd33a6086c4 node DatanodeRegistration(127.0.0.1:38583, datanodeUuid=1e03318e-cf1d-4f81-b62e-69f76a2f240a, infoPort=38051, infoSecurePort=0, ipcPort=43547, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,749 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ff337dfb85ec321 with lease ID 0x83a199dfabd8ed0d: Processing first storage report for DS-1cc84279-7585-4d43-992d-3bc36825540d from datanode DatanodeRegistration(127.0.0.1:38583, datanodeUuid=1e03318e-cf1d-4f81-b62e-69f76a2f240a, infoPort=38051, infoSecurePort=0, ipcPort=43547, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232)
2024-11-19T07:34:29,749 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ff337dfb85ec321 with lease ID 0x83a199dfabd8ed0d: from storage DS-1cc84279-7585-4d43-992d-3bc36825540d node DatanodeRegistration(127.0.0.1:38583, datanodeUuid=1e03318e-cf1d-4f81-b62e-69f76a2f240a, infoPort=38051, infoSecurePort=0, ipcPort=43547, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:29,784 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,784 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,784 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,784 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,785 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:29,785 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:29,785 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:29,785 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta block BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:29,785 WARN  [PacketResponder: BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35925] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:58596 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44027:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58596 dst: /127.0.0.1:44027
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:58572 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44027:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58572 dst: /127.0.0.1:44027
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166448768_22 at /127.0.0.1:57710 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35925:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57710 dst: /127.0.0.1:35925
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166448768_22 at /127.0.0.1:58518 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44027:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58518 dst: /127.0.0.1:44027
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:58562 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44027:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58562 dst: /127.0.0.1:44027
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:57750 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35925:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57750 dst: /127.0.0.1:35925
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:57740 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35925:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57740 dst: /127.0.0.1:35925
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,786 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:57770 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:35925:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57770 dst: /127.0.0.1:35925
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,789 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6cd7b3e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:29,790 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ff95875{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:29,790 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:29,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32403ac6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:29,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f7f19bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:29,791 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:29,791 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:29,791 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:29,791 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 4c0ab2de-fd04-4658-ad1b-d728f7cc4f29) service to localhost/127.0.0.1:40659
2024-11-19T07:34:29,792 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data3/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:29,792 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data4/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:29,792 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:29,797 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@6e4ede9d {}] datanode.DataXceiver(331): 127.0.0.1:44027:DataXceiver error processing unknown operation  src: /127.0.0.1:52284 dst: /127.0.0.1:44027
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:29,797 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,798 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,800 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta block BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,800 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,801 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c77eea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:29,802 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e20426d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:29,802 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:29,802 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf32f74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:29,802 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2cb9bebc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:29,803 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:29,803 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:29,803 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 93814c7a-5f63-4340-a4d2-45d183aecc07) service to localhost/127.0.0.1:40659
2024-11-19T07:34:29,803 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:29,804 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data1/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:29,804 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data2/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:29,804 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., hostname=db0afee3eab9,37853,1732001657436, seqNum=2]
2024-11-19T07:34:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:34:29,814 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:34:29,815 ERROR [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332-prefix:db0afee3eab9,37853,1732001657436 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,815 WARN  [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332-prefix:db0afee3eab9,37853,1732001657436 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,815 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C37853%2C1732001657436:(num 1732001657673) roll requested
2024-11-19T07:34:29,816 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001669816
2024-11-19T07:34:29,826 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:29,826 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:29,826 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:29,826 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:29,826 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:29,826 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816
2024-11-19T07:34:29,827 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,827 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:29,828 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-19T07:34:29,828 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-19T07:34:29,829 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673
2024-11-19T07:34:29,829 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38051:38051),(127.0.0.1/127.0.0.1:36355:36355)]
2024-11-19T07:34:29,829 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:29,832 WARN  [IPC Server handler 4 on default port 40659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741837_1013
2024-11-19T07:34:29,836 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 after 5ms
2024-11-19T07:34:30,293 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:30,667 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:30,668 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:31,831 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816
2024-11-19T07:34:31,831 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:31,832 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:31,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:53918 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:38583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53918 dst: /127.0.0.1:38583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:31,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:35262 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:42413:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35262 dst: /127.0.0.1:42413
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:31,933 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@33f262d4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:31,933 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44692c68{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:31,933 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:31,933 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264a677b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:31,934 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5455501c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:31,935 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:31,935 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:31,936 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:31,936 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 1e03318e-cf1d-4f81-b62e-69f76a2f240a) service to localhost/127.0.0.1:40659
2024-11-19T07:34:31,936 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data9/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:31,936 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data10/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:31,937 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:32,293 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:32,668 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:32,668 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:33,830 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]
2024-11-19T07:34:33,830 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C37853%2C1732001657436:(num 1732001669816) roll requested
2024-11-19T07:34:33,830 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001673830
2024-11-19T07:34:33,836 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44027
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:33,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:36254 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data6]'}, localName='127.0.0.1:42413', datanodeUuid='66f79c7b-09c8-4374-9926-17250bcc504d', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021 to mirror 127.0.0.1:44027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:33,836 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:33,836 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021
2024-11-19T07:34:33,836 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:36254 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:33,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:36254 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42413:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36254 dst: /127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:33,837 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 after 4008ms
2024-11-19T07:34:33,839 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:33,847 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:33,847 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:33,847 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:33,847 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:33,847 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:33,847 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001673830
2024-11-19T07:34:33,848 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36355:36355),(127.0.0.1/127.0.0.1:40565:40565)]
2024-11-19T07:34:33,848 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:33,848 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816 is not closed yet, will try archiving it next time
2024-11-19T07:34:33,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42413 is added to blk_1073741838_1020 (size=2431)
2024-11-19T07:34:33,942 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-19T07:34:34,251 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:34,294 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:34,668 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:34,669 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@30814498[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42413, datanodeUuid=66f79c7b-09c8-4374-9926-17250bcc504d, infoPort=36355, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741838_1020 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:35,947 WARN  [ResponseProcessor for block BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,947 WARN  [DataStreamer for file /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001673830 block BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:35,947 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:36262 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:42413:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36262 dst: /127.0.0.1:42413
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:35,948 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33282 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33282 dst: /127.0.0.1:39757
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:35,949 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15751333{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:35,950 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@184b7b23{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:34:35,950 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:34:35,950 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cac6b83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:34:35,950 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45d50f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:34:35,952 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:34:35,952 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:34:35,952 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:34:35,952 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 66f79c7b-09c8-4374-9926-17250bcc504d) service to localhost/127.0.0.1:40659
2024-11-19T07:34:35,952 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data5/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:35,952 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data6/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:34:35,953 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:34:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:35,963 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:34:35,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/68c7cf3808ec434faed71dab81da49f7 is 1080, key is row0002/info:/1732001671938/Put/seqid=0
2024-11-19T07:34:35,985 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,985 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:35,985 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741841_1024
2024-11-19T07:34:35,986 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:35,987 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,987 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:35,988 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741842_1025
2024-11-19T07:34:35,993 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:35,994 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,994 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:35,994 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741843_1026
2024-11-19T07:34:35,995 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:35,996 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:35,997 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:35,997 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741844_1027
2024-11-19T07:34:35,997 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:35,998 WARN  [IPC Server handler 0 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:35,999 WARN  [IPC Server handler 0 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:35,999 WARN  [IPC Server handler 0 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:36,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741845_1028 (size=10347)
2024-11-19T07:34:36,294 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,404 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/68c7cf3808ec434faed71dab81da49f7
2024-11-19T07:34:36,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/68c7cf3808ec434faed71dab81da49f7 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7
2024-11-19T07:34:36,419 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7, entries=5, sequenceid=11, filesize=10.1 K
2024-11-19T07:34:36,421 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 1336ae17c9e49afba8075a95cc1255a1 in 457ms, sequenceid=11, compaction requested=false
2024-11-19T07:34:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:36,589 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-19T07:34:36,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/5f657125ced74ae59fa42192d4e0d95b is 1080, key is row0007/info:/1732001675964/Put/seqid=0
2024-11-19T07:34:36,597 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,597 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:36,597 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741846_1029
2024-11-19T07:34:36,598 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:36,602 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33328 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,602 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:36,602 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030
2024-11-19T07:34:36,602 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33328 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:36,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33328 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33328 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,603 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:36,606 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,606 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33336 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031 to mirror 127.0.0.1:38583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,606 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:36,606 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031
2024-11-19T07:34:36,606 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33336 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:36,607 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33336 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33336 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,607 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:36,609 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35925
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33344 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032 to mirror 127.0.0.1:35925
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,610 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:36,610 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032
2024-11-19T07:34:36,610 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33344 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:36,610 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33344 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33344 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:36,610 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:36,611 WARN  [IPC Server handler 0 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:36,611 WARN  [IPC Server handler 0 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:36,611 WARN  [IPC Server handler 0 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:36,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741850_1033 (size=12506)
2024-11-19T07:34:36,669 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:36,669 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:37,015 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/5f657125ced74ae59fa42192d4e0d95b
2024-11-19T07:34:37,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/5f657125ced74ae59fa42192d4e0d95b as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b
2024-11-19T07:34:37,032 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b, entries=7, sequenceid=24, filesize=12.2 K
2024-11-19T07:34:37,033 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 1336ae17c9e49afba8075a95cc1255a1 in 443ms, sequenceid=24, compaction requested=false
2024-11-19T07:34:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-19T07:34:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b because midkey is the same as first or last row
2024-11-19T07:34:37,849 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]
2024-11-19T07:34:37,849 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C37853%2C1732001657436:(num 1732001673830) roll requested
2024-11-19T07:34:37,850 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001677849
2024-11-19T07:34:37,854 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:37,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33364 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034 to mirror 127.0.0.1:38583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,854 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:37,854 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034
2024-11-19T07:34:37,854 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33364 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:37,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33364 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33364 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,855 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:37,858 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44027
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:37,858 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33372 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035 to mirror 127.0.0.1:44027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,858 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:37,858 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035
2024-11-19T07:34:37,858 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33372 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:37,858 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33372 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33372 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,859 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:37,861 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:37,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33386 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,862 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:37,862 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33386 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:37,862 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036
2024-11-19T07:34:37,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33386 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33386 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,863 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:37,865 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35925
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:37,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33392 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037 to mirror 127.0.0.1:35925
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,866 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:37,866 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33392 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:37,866 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037
2024-11-19T07:34:37,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33392 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33392 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:37,866 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:37,867 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:37,867 WARN  [IPC Server handler 1 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:37,867 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:37,871 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:37,871 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:37,871 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:37,871 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:37,871 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:37,871 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001673830 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001677849
2024-11-19T07:34:37,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741840_1023 (size=25992)
2024-11-19T07:34:37,875 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40565:40565)]
2024-11-19T07:34:37,875 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:37,875 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001673830 is not closed yet, will try archiving it next time
2024-11-19T07:34:37,876 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001669816 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs/db0afee3eab9%2C37853%2C1732001657436.1732001669816
2024-11-19T07:34:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:38,009 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T07:34:38,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/75e49e0f74f04c658f360262bfe95f5a is 1079, key is tmprow/info:/1732001678008/Put/seqid=0
2024-11-19T07:34:38,018 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,019 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:38,019 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741856_1039
2024-11-19T07:34:38,019 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:38,021 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,021 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:38,021 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741857_1040
2024-11-19T07:34:38,021 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:38,022 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,023 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:38,023 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741858_1041
2024-11-19T07:34:38,023 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:38,024 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,025 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:38,025 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741859_1042
2024-11-19T07:34:38,025 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:38,026 WARN  [IPC Server handler 4 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:38,026 WARN  [IPC Server handler 4 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:38,026 WARN  [IPC Server handler 4 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:38,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741860_1043 (size=6027)
2024-11-19T07:34:38,274 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:38,295 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,430 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/75e49e0f74f04c658f360262bfe95f5a
2024-11-19T07:34:38,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/75e49e0f74f04c658f360262bfe95f5a as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a
2024-11-19T07:34:38,448 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a, entries=1, sequenceid=34, filesize=5.9 K
2024-11-19T07:34:38,449 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 1336ae17c9e49afba8075a95cc1255a1 in 440ms, sequenceid=34, compaction requested=true
2024-11-19T07:34:38,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:38,450 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-19T07:34:38,450 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:38,450 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b because midkey is the same as first or last row
2024-11-19T07:34:38,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1336ae17c9e49afba8075a95cc1255a1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:34:38,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:38,450 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:34:38,451 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:34:38,451 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1541): 1336ae17c9e49afba8075a95cc1255a1/info is initiating minor compaction (all files)
2024-11-19T07:34:38,452 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1336ae17c9e49afba8075a95cc1255a1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:38,452 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a] into tmpdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp, totalSize=28.2 K
2024-11-19T07:34:38,452 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c7cf3808ec434faed71dab81da49f7, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732001671938
2024-11-19T07:34:38,453 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f657125ced74ae59fa42192d4e0d95b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732001675964
2024-11-19T07:34:38,453 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75e49e0f74f04c658f360262bfe95f5a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732001678008
2024-11-19T07:34:38,468 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1336ae17c9e49afba8075a95cc1255a1#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:34:38,469 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/79b0c1dab5b84c5b9269b8c06abd73da is 1080, key is row0002/info:/1732001671938/Put/seqid=0
2024-11-19T07:34:38,470 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,471 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:38,471 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741861_1044
2024-11-19T07:34:38,471 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:38,473 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,473 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:38,473 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741862_1045
2024-11-19T07:34:38,474 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:38,476 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,476 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33420 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:38,476 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:38,476 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046
2024-11-19T07:34:38,476 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33420 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:38,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33420 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33420 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:38,477 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:38,479 WARN  [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,479 WARN  [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:38,479 WARN  [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741864_1047
2024-11-19T07:34:38,480 WARN  [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:38,480 WARN  [IPC Server handler 2 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:38,480 WARN  [IPC Server handler 2 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:38,481 WARN  [IPC Server handler 2 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:38,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741865_1048 (size=17994)
2024-11-19T07:34:38,491 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/79b0c1dab5b84c5b9269b8c06abd73da as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da
2024-11-19T07:34:38,499 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1336ae17c9e49afba8075a95cc1255a1/info of 1336ae17c9e49afba8075a95cc1255a1 into 79b0c1dab5b84c5b9269b8c06abd73da(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:38,499 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., storeName=1336ae17c9e49afba8075a95cc1255a1/info, priority=13, startTime=1732001678450; duration=0sec
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da because midkey is the same as first or last row
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:38,499 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da because midkey is the same as first or last row
2024-11-19T07:34:38,500 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-19T07:34:38,500 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:38,500 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da because midkey is the same as first or last row
2024-11-19T07:34:38,500 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:38,500 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1336ae17c9e49afba8075a95cc1255a1:info
2024-11-19T07:34:38,641 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4b478627[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741845_1028 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:38,641 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4580713f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741850_1033 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:38,669 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:38,669 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:39,431 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T07:34:39,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/b7052b6f48e44249b84964b1d16a4735 is 1079, key is tmprow/info:/1732001679429/Put/seqid=0
2024-11-19T07:34:39,440 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,440 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:39,440 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741866_1049
2024-11-19T07:34:39,441 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:39,442 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,442 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:39,442 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741867_1050
2024-11-19T07:34:39,443 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:39,444 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,444 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:39,444 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741868_1051
2024-11-19T07:34:39,445 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:39,446 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,447 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:39,447 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741869_1052
2024-11-19T07:34:39,447 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:39,448 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:39,448 WARN  [IPC Server handler 1 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:39,448 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:39,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741870_1053 (size=6027)
2024-11-19T07:34:39,641 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4b478627[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741860_1043 to 127.0.0.1:35925 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,641 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4580713f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741840_1023 to 127.0.0.1:44027 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,852 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/b7052b6f48e44249b84964b1d16a4735
2024-11-19T07:34:39,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/b7052b6f48e44249b84964b1d16a4735 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735
2024-11-19T07:34:39,871 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735, entries=1, sequenceid=45, filesize=5.9 K
2024-11-19T07:34:39,873 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 1336ae17c9e49afba8075a95cc1255a1 in 441ms, sequenceid=45, compaction requested=false
2024-11-19T07:34:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-19T07:34:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:39,873 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da because midkey is the same as first or last row
2024-11-19T07:34:39,876 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]
2024-11-19T07:34:39,876 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C37853%2C1732001657436:(num 1732001677849) roll requested
2024-11-19T07:34:39,876 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001679876
2024-11-19T07:34:39,879 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,879 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:39,879 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741871_1054
2024-11-19T07:34:39,880 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:39,881 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,881 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:39,881 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741872_1055
2024-11-19T07:34:39,882 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:39,884 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44027
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33448 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056 to mirror 127.0.0.1:44027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,884 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33448 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:39,884 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:39,884 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056
2024-11-19T07:34:39,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33448 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33448 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,885 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:39,887 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:39,887 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33450 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057 to mirror 127.0.0.1:38583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,888 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:39,888 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33450 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:39,888 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057
2024-11-19T07:34:39,888 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33450 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33450 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:39,888 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:39,889 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:39,889 WARN  [IPC Server handler 1 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:39,889 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:39,892 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:39,892 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:39,892 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:39,892 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:39,892 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:39,892 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001677849 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001679876
2024-11-19T07:34:39,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741855_1038 (size=13591)
2024-11-19T07:34:39,896 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40565:40565)]
2024-11-19T07:34:39,896 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:39,896 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001677849 is not closed yet, will try archiving it next time
2024-11-19T07:34:39,896 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001673830 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs/db0afee3eab9%2C37853%2C1732001657436.1732001673830
2024-11-19T07:34:40,295 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 is not closed yet, will try archiving it next time
2024-11-19T07:34:40,295 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,670 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,670 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:40,854 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T07:34:40,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/bf65e5921a1746bdb1dce1ac43582a52 is 1079, key is tmprow/info:/1732001680853/Put/seqid=0
2024-11-19T07:34:40,862 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,863 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:40,863 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741876_1059
2024-11-19T07:34:40,863 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:40,865 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,865 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:40,865 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741877_1060
2024-11-19T07:34:40,866 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:40,867 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,868 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:40,868 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741878_1061
2024-11-19T07:34:40,868 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:40,871 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35925
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,871 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33466 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062 to mirror 127.0.0.1:35925
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:40,871 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:40,871 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062
2024-11-19T07:34:40,871 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33466 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:40,871 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33466 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33466 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:40,872 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:40,873 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:40,873 WARN  [IPC Server handler 1 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:40,873 WARN  [IPC Server handler 1 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:40,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741880_1063 (size=6027)
2024-11-19T07:34:40,877 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/bf65e5921a1746bdb1dce1ac43582a52
2024-11-19T07:34:40,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/bf65e5921a1746bdb1dce1ac43582a52 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52
2024-11-19T07:34:40,890 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52, entries=1, sequenceid=55, filesize=5.9 K
2024-11-19T07:34:40,891 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 1336ae17c9e49afba8075a95cc1255a1 in 36ms, sequenceid=55, compaction requested=true
2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da because midkey is the same as first or last row
2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1336ae17c9e49afba8075a95cc1255a1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:34:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:40,891 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:34:40,892 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:34:40,892 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1541): 1336ae17c9e49afba8075a95cc1255a1/info is initiating minor compaction (all files)
2024-11-19T07:34:40,893 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1336ae17c9e49afba8075a95cc1255a1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:40,893 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52] into tmpdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp, totalSize=29.3 K
2024-11-19T07:34:40,893 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b0c1dab5b84c5b9269b8c06abd73da, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732001671938
2024-11-19T07:34:40,893 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7052b6f48e44249b84964b1d16a4735, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732001679429
2024-11-19T07:34:40,894 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf65e5921a1746bdb1dce1ac43582a52, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732001680853
2024-11-19T07:34:40,910 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1336ae17c9e49afba8075a95cc1255a1#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:34:40,911 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/8191ae9da1564b289b59fd98b3dd35c4 is 1080, key is row0002/info:/1732001671938/Put/seqid=0
2024-11-19T07:34:40,912 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,912 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:40,913 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741881_1064
2024-11-19T07:34:40,913 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:40,915 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,915 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33476 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065 to mirror 127.0.0.1:38583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:40,915 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]) is bad.
2024-11-19T07:34:40,915 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065
2024-11-19T07:34:40,916 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33476 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065] {}] datanode.BlockReceiver(316): Block 1073741882 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:40,916 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:33476 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741882_1065] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33476 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:40,916 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38583,DS-fb9392d4-064c-4a30-a923-0bd33a6086c4,DISK]
2024-11-19T07:34:40,918 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,918 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]) is bad.
2024-11-19T07:34:40,918 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741883_1066
2024-11-19T07:34:40,918 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35925,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]
2024-11-19T07:34:40,920 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:40,920 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]) is bad.
2024-11-19T07:34:40,920 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741884_1067
2024-11-19T07:34:40,921 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]
2024-11-19T07:34:40,921 WARN  [IPC Server handler 2 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-19T07:34:40,921 WARN  [IPC Server handler 2 on default port 40659 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-19T07:34:40,921 WARN  [IPC Server handler 2 on default port 40659 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-19T07:34:40,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741885_1068 (size=18097)
2024-11-19T07:34:41,333 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/8191ae9da1564b289b59fd98b3dd35c4 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4
2024-11-19T07:34:41,341 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1336ae17c9e49afba8075a95cc1255a1/info of 1336ae17c9e49afba8075a95cc1255a1 into 8191ae9da1564b289b59fd98b3dd35c4(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:34:41,341 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:41,342 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., storeName=1336ae17c9e49afba8075a95cc1255a1/info, priority=13, startTime=1732001680891; duration=0sec
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 because midkey is the same as first or last row
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 because midkey is the same as first or last row
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 because midkey is the same as first or last row
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:41,342 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1336ae17c9e49afba8075a95cc1255a1:info
2024-11-19T07:34:41,642 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4b478627[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741870_1053 to 127.0.0.1:35925 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:41,642 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4580713f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741865_1048 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:41,897 WARN  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-19T07:34:42,082 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:34:42,087 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:34:42,088 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:34:42,088 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:34:42,088 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:34:42,089 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d6a3337{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:34:42,089 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a8eeeb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:34:42,190 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@122a196d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/java.io.tmpdir/jetty-localhost-37101-hadoop-hdfs-3_4_1-tests_jar-_-any-7035726118769830175/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:34:42,191 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a528add{HTTP/1.1, (http/1.1)}{localhost:37101}
2024-11-19T07:34:42,191 INFO  [Time-limited test {}] server.Server(415): Started @138880ms
2024-11-19T07:34:42,192 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:34:42,295 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:42,643 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4580713f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741855_1038 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:42,643 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4b478627[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39757, datanodeUuid=70018873-12a4-426e-bade-060972dd72d8, infoPort=40565, infoSecurePort=0, ipcPort=34923, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741880_1063 to 127.0.0.1:35925 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:42,665 WARN  [Thread-993 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:34:42,670 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:42,670 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:42,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20f19df1d9e1a461 with lease ID 0x83a199dfabd8ed0e: from storage DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3 node DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T07:34:42,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20f19df1d9e1a461 with lease ID 0x83a199dfabd8ed0e: from storage DS-f1c1ee06-b282-494f-bbf3-95152a77ee1e node DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:34:44,296 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:44,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741885_1068 (size=18097)
2024-11-19T07:34:44,671 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:44,671 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:45,983 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:34:46,296 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,526 ERROR [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData-prefix:db0afee3eab9,42985,1732001656005 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,526 WARN  [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData-prefix:db0afee3eab9,42985,1732001656005 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,526 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C42985%2C1732001656005:(num 1732001656313) roll requested
2024-11-19T07:34:46,527 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42985%2C1732001656005.1732001686526
2024-11-19T07:34:46,533 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:46,533 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:46,533 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:46,533 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:46,533 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:46,534 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001686526
2024-11-19T07:34:46,534 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,534 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,535 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313
2024-11-19T07:34:46,535 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39479:39479),(127.0.0.1/127.0.0.1:40565:40565)]
2024-11-19T07:34:46,535 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 is not closed yet, will try archiving it next time
2024-11-19T07:34:46,535 WARN  [IPC Server handler 4 on default port 40659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 has not been closed. Lease recovery is in progress. RecoveryId = 1070 for block blk_1073741830_1006
2024-11-19T07:34:46,535 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 after 0ms
2024-11-19T07:34:46,671 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:46,671 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:48,671 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:48,672 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:50,537 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/WALs/db0afee3eab9,42985,1732001656005/db0afee3eab9%2C42985%2C1732001656005.1732001656313 after 4002ms
2024-11-19T07:34:50,669 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3ac44c4d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741835_1011 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:50,670 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d222026[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741831_1007 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:50,672 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:50,672 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:51,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:34:51,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:34:52,672 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:52,673 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:52,687 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@b85a56d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:44027,null,null])
java.net.ConnectException: Call From db0afee3eab9/172.17.0.2 to localhost:39579 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T07:34:52,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741837_1019 (size=455)
2024-11-19T07:34:52,854 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001657673 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs/db0afee3eab9%2C37853%2C1732001657436.1732001657673
2024-11-19T07:34:52,856 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001677849 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs/db0afee3eab9%2C37853%2C1732001657436.1732001677849
2024-11-19T07:34:53,671 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d222026[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741837_1019 to 127.0.0.1:38583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:53,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:34:54,673 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:54,673 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:55,322 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37853%2C1732001657436.1732001695322
2024-11-19T07:34:55,325 WARN  [Thread-1028 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1071
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:55,325 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166448768_22 at /127.0.0.1:52664 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:55,325 WARN  [Thread-1028 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:55,325 WARN  [Thread-1028 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071
2024-11-19T07:34:55,325 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-166448768_22 at /127.0.0.1:52664 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-19T07:34:55,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166448768_22 at /127.0.0.1:52664 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741887_1071] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52664 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:55,326 WARN  [Thread-1028 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:55,330 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:55,330 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:55,331 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:55,331 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:55,331 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:55,331 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001679876 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001695322
2024-11-19T07:34:55,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40565:40565),(127.0.0.1/127.0.0.1:39479:39479)]
2024-11-19T07:34:55,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001679876 is not closed yet, will try archiving it next time
2024-11-19T07:34:55,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741875_1058 (size=12911)
2024-11-19T07:34:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:55,337 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-19T07:34:55,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/8cc9f92476ca4b42ba164067097e8fb0 is 1080, key is row0013/info:/1732001695333/Put/seqid=0
2024-11-19T07:34:55,347 WARN  [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:55,347 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:34714 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data4]'}, localName='127.0.0.1:35263', datanodeUuid='4c0ab2de-fd04-4658-ad1b-d728f7cc4f29', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:55,347 WARN  [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35263,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:55,347 WARN  [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073
2024-11-19T07:34:55,347 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:34714 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073] {}] datanode.BlockReceiver(316): Block 1073741889 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:55,347 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-834007017_22 at /127.0.0.1:34714 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741889_1073] {}] datanode.DataXceiver(331): 127.0.0.1:35263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34714 dst: /127.0.0.1:35263
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:55,356 WARN  [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:55,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741890_1074 (size=8190)
2024-11-19T07:34:55,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741890_1074 (size=8190)
2024-11-19T07:34:55,366 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/8cc9f92476ca4b42ba164067097e8fb0
2024-11-19T07:34:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/8cc9f92476ca4b42ba164067097e8fb0 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0
2024-11-19T07:34:55,383 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0, entries=3, sequenceid=66, filesize=8.0 K
2024-11-19T07:34:55,384 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 1336ae17c9e49afba8075a95cc1255a1 in 47ms, sequenceid=66, compaction requested=false
2024-11-19T07:34:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-19T07:34:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 because midkey is the same as first or last row
2024-11-19T07:34:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37853 {}] regionserver.HRegion(8855): Flush requested on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:55,564 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-19T07:34:55,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/16bf97f343b74819b70467d34aac9c14 is 1080, key is row0015/info:/1732001695339/Put/seqid=0
2024-11-19T07:34:55,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741891_1075 (size=14660)
2024-11-19T07:34:55,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741891_1075 (size=14660)
2024-11-19T07:34:55,583 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/16bf97f343b74819b70467d34aac9c14
2024-11-19T07:34:55,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/16bf97f343b74819b70467d34aac9c14 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14
2024-11-19T07:34:55,597 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14, entries=9, sequenceid=79, filesize=14.3 K
2024-11-19T07:34:55,598 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 1336ae17c9e49afba8075a95cc1255a1 in 34ms, sequenceid=79, compaction requested=true
2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 because midkey is the same as first or last row
2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1336ae17c9e49afba8075a95cc1255a1:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:34:55,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:55,598 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:34:55,600 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:34:55,600 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1541): 1336ae17c9e49afba8075a95cc1255a1/info is initiating minor compaction (all files)
2024-11-19T07:34:55,600 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1336ae17c9e49afba8075a95cc1255a1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:55,600 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14] into tmpdir=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp, totalSize=40.0 K
2024-11-19T07:34:55,600 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8191ae9da1564b289b59fd98b3dd35c4, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732001671938
2024-11-19T07:34:55,601 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc9f92476ca4b42ba164067097e8fb0, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732001681870
2024-11-19T07:34:55,601 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16bf97f343b74819b70467d34aac9c14, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732001695339
2024-11-19T07:34:55,614 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1336ae17c9e49afba8075a95cc1255a1#info#compaction#27 average throughput is 22.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:34:55,615 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/02646e3567c14fc985a48c44885586ea is 1080, key is row0002/info:/1732001671938/Put/seqid=0
2024-11-19T07:34:55,617 WARN  [Thread-1051 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:55,617 WARN  [Thread-1051 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:55,617 WARN  [Thread-1051 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741892_1076
2024-11-19T07:34:55,618 WARN  [Thread-1051 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:55,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741893_1077 (size=28989)
2024-11-19T07:34:55,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741893_1077 (size=28989)
2024-11-19T07:34:55,630 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/02646e3567c14fc985a48c44885586ea as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/02646e3567c14fc985a48c44885586ea
2024-11-19T07:34:55,637 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1336ae17c9e49afba8075a95cc1255a1/info of 1336ae17c9e49afba8075a95cc1255a1 into 02646e3567c14fc985a48c44885586ea(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:34:55,637 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1336ae17c9e49afba8075a95cc1255a1:

2024-11-19T07:34:55,637 INFO  [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., storeName=1336ae17c9e49afba8075a95cc1255a1/info, priority=13, startTime=1732001695598; duration=0sec
2024-11-19T07:34:55,637 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T07:34:55,637 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:55,637 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/02646e3567c14fc985a48c44885586ea because midkey is the same as first or last row
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/02646e3567c14fc985a48c44885586ea because midkey is the same as first or last row
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/02646e3567c14fc985a48c44885586ea because midkey is the same as first or last row
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:34:55,638 DEBUG [RS:1;db0afee3eab9:37853-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1336ae17c9e49afba8075a95cc1255a1:info
2024-11-19T07:34:55,734 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,37853,1732001657436/db0afee3eab9%2C37853%2C1732001657436.1732001679876 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs/db0afee3eab9%2C37853%2C1732001657436.1732001679876
2024-11-19T07:34:55,897 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-19T07:34:55,968 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:34:55,968 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:55,968 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:55,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:55,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:55,968 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:34:55,968 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=86293236, stopped=false
2024-11-19T07:34:55,969 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,42985,1732001656005
2024-11-19T07:34:55,969 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:34:56,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:56,039 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:56,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:34:56,039 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:56,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:56,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:34:56,039 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:34:56,039 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:34:56,039 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:56,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:56,040 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,42239,1732001656169' *****
2024-11-19T07:34:56,040 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:34:56,040 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,37853,1732001657436' *****
2024-11-19T07:34:56,040 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:34:56,040 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:34:56,040 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:34:56,040 INFO  [RS:1;db0afee3eab9:37853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:34:56,040 INFO  [RS:0;db0afee3eab9:42239 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:34:56,040 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:34:56,040 INFO  [RS:1;db0afee3eab9:37853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:34:56,040 INFO  [RS:0;db0afee3eab9:42239 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:34:56,040 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,42239,1732001656169
2024-11-19T07:34:56,040 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:56,041 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(3091): Received CLOSE for 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:56,041 INFO  [RS:0;db0afee3eab9:42239 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:42239.
2024-11-19T07:34:56,041 DEBUG [RS:0;db0afee3eab9:42239 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:56,041 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:34:56,041 DEBUG [RS:0;db0afee3eab9:42239 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:56,041 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:34:56,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:56,041 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:34:56,041 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:34:56,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:56,041 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:34:56,041 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,37853,1732001657436
2024-11-19T07:34:56,041 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:34:56,042 INFO  [RS:1;db0afee3eab9:37853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;db0afee3eab9:37853.
2024-11-19T07:34:56,042 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:34:56,042 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T07:34:56,042 DEBUG [RS:1;db0afee3eab9:37853 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1336ae17c9e49afba8075a95cc1255a1, disabling compactions & flushes
2024-11-19T07:34:56,042 DEBUG [RS:1;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:56,042 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-19T07:34:56,042 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:56,042 DEBUG [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T07:34:56,042 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T07:34:56,042 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1325): Online Regions={1336ae17c9e49afba8075a95cc1255a1=TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.}
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:56,042 DEBUG [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1351): Waiting on 1336ae17c9e49afba8075a95cc1255a1
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1. after waiting 0 ms
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:34:56,042 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:34:56,042 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:34:56,042 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1336ae17c9e49afba8075a95cc1255a1 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:34:56,042 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-19T07:34:56,043 ERROR [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332-prefix:db0afee3eab9,42239,1732001656169.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,043 WARN  [FSHLog-0-hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332-prefix:db0afee3eab9,42239,1732001656169.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,043 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,043 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C42239%2C1732001656169.meta:.meta(num 1732001657163) roll requested
2024-11-19T07:34:56,044 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42239%2C1732001656169.meta.1732001696043.meta
2024-11-19T07:34:56,046 WARN  [Thread-1057 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,047 WARN  [Thread-1057 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK], DatanodeInfoWithStorage[127.0.0.1:35263,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:56,047 WARN  [Thread-1057 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741894_1078
2024-11-19T07:34:56,047 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/08bdf5a79de54aee9f6dc87f069736a6 is 1079, key is tmprow/info:/1732001695766/Put/seqid=0
2024-11-19T07:34:56,047 WARN  [Thread-1057 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:56,056 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,056 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,056 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,056 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,056 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,057 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001696043.meta
2024-11-19T07:34:56,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741896_1080 (size=6027)
2024-11-19T07:34:56,057 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741896_1080 (size=6027)
2024-11-19T07:34:56,057 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,057 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
2024-11-19T07:34:56,058 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40565:40565),(127.0.0.1/127.0.0.1:39479:39479)]
2024-11-19T07:34:56,058 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta is not closed yet, will try archiving it next time
2024-11-19T07:34:56,058 WARN  [IPC Server handler 0 on default port 40659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010
2024-11-19T07:34:56,058 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/08bdf5a79de54aee9f6dc87f069736a6
2024-11-19T07:34:56,058 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta after 1ms
2024-11-19T07:34:56,065 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/.tmp/info/08bdf5a79de54aee9f6dc87f069736a6 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/08bdf5a79de54aee9f6dc87f069736a6
2024-11-19T07:34:56,072 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/08bdf5a79de54aee9f6dc87f069736a6, entries=1, sequenceid=84, filesize=5.9 K
2024-11-19T07:34:56,074 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1336ae17c9e49afba8075a95cc1255a1 in 31ms, sequenceid=84, compaction requested=false
2024-11-19T07:34:56,076 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14] to archive
2024-11-19T07:34:56,078 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:34:56,080 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/68c7cf3808ec434faed71dab81da49f7
2024-11-19T07:34:56,081 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/info/1f3348e89d544bb38b8d8e8f8560fbb2 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1./info:regioninfo/1732001658355/Put/seqid=0
2024-11-19T07:34:56,082 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/5f657125ced74ae59fa42192d4e0d95b
2024-11-19T07:34:56,083 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/79b0c1dab5b84c5b9269b8c06abd73da
2024-11-19T07:34:56,085 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1082
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,085 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:52732 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8]'}, localName='127.0.0.1:39757', datanodeUuid='70018873-12a4-426e-bade-060972dd72d8', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:56,085 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39757,DS-9a4a5a6d-6bbd-4cc8-b81a-2a1c218215af,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:56,085 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082
2024-11-19T07:34:56,085 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:52732 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:56,086 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:52732 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741897_1082] {}] datanode.DataXceiver(331): 127.0.0.1:39757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52732 dst: /127.0.0.1:39757
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:56,086 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:56,086 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/75e49e0f74f04c658f360262bfe95f5a
2024-11-19T07:34:56,088 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/b7052b6f48e44249b84964b1d16a4735
2024-11-19T07:34:56,091 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8191ae9da1564b289b59fd98b3dd35c4
2024-11-19T07:34:56,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741898_1083 (size=7089)
2024-11-19T07:34:56,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741898_1083 (size=7089)
2024-11-19T07:34:56,093 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/bf65e5921a1746bdb1dce1ac43582a52
2024-11-19T07:34:56,094 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/info/1f3348e89d544bb38b8d8e8f8560fbb2
2024-11-19T07:34:56,095 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/8cc9f92476ca4b42ba164067097e8fb0
2024-11-19T07:34:56,097 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14 to hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/info/16bf97f343b74819b70467d34aac9c14
2024-11-19T07:34:56,097 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:42985 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-19T07:34:56,098 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [68c7cf3808ec434faed71dab81da49f7=10347, 5f657125ced74ae59fa42192d4e0d95b=12506, 79b0c1dab5b84c5b9269b8c06abd73da=17994, 75e49e0f74f04c658f360262bfe95f5a=6027, b7052b6f48e44249b84964b1d16a4735=6027, 8191ae9da1564b289b59fd98b3dd35c4=18097, bf65e5921a1746bdb1dce1ac43582a52=6027, 8cc9f92476ca4b42ba164067097e8fb0=8190, 16bf97f343b74819b70467d34aac9c14=14660]
2024-11-19T07:34:56,103 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1336ae17c9e49afba8075a95cc1255a1/recovered.edits/87.seqid, newMaxSeqId=87, maxSeqId=1
2024-11-19T07:34:56,103 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:56,103 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1336ae17c9e49afba8075a95cc1255a1:
Waiting for close lock at 1732001696042Running coprocessor pre-close hooks at 1732001696042Disabling compacts and flushes for region at 1732001696042Disabling writes for close at 1732001696042Obtaining lock to block concurrent updates at 1732001696042Preparing flush snapshotting stores in 1336ae17c9e49afba8075a95cc1255a1 at 1732001696042Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732001696043 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1. at 1732001696043Flushing 1336ae17c9e49afba8075a95cc1255a1/info: creating writer at 1732001696044 (+1 ms)Flushing 1336ae17c9e49afba8075a95cc1255a1/info: appending metadata at 1732001696047 (+3 ms)Flushing 1336ae17c9e49afba8075a95cc1255a1/info: closing flushed file at 1732001696047Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5646e5ab: reopening flushed file at 1732001696064 (+17 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1336ae17c9e49afba8075a95cc1255a1 in 31ms, sequenceid=84, compaction requested=false at 1732001696074 (+10 ms)Writing region close event to WAL at 1732001696098 (+24 ms)Running coprocessor post-close hooks at 1732001696103 (+5 ms)Closed at 1732001696103
2024-11-19T07:34:56,104 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732001657568.1336ae17c9e49afba8075a95cc1255a1.
2024-11-19T07:34:56,118 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/ns/f2cbbb3e1d4a4d03ae2ced267aa5900b is 43, key is default/ns:d/1732001657319/Put/seqid=0
2024-11-19T07:34:56,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741899_1084 (size=5153)
2024-11-19T07:34:56,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741899_1084 (size=5153)
2024-11-19T07:34:56,124 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/ns/f2cbbb3e1d4a4d03ae2ced267aa5900b
2024-11-19T07:34:56,149 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/table/097a65b540ad447281b0f215e2832207 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732001658368/Put/seqid=0
2024-11-19T07:34:56,152 WARN  [Thread-1083 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1085
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42413
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:56,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:34758 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data4]'}, localName='127.0.0.1:35263', datanodeUuid='4c0ab2de-fd04-4658-ad1b-d728f7cc4f29', xmitsInProgress=0}:Exception transferring block BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085 to mirror 127.0.0.1:42413
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:56,152 WARN  [Thread-1083 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35263,DS-5ec3a1cc-3109-41ad-b738-3b2e235092b3,DISK], DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]) is bad.
2024-11-19T07:34:56,152 WARN  [Thread-1083 {}] hdfs.DataStreamer(1850): Abandoning BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085
2024-11-19T07:34:56,152 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:34758 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085] {}] datanode.BlockReceiver(316): Block 1073741900 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-19T07:34:56,153 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_368250562_22 at /127.0.0.1:34758 [Receiving block BP-1194886781-172.17.0.2-1732001653232:blk_1073741900_1085] {}] datanode.DataXceiver(331): 127.0.0.1:35263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34758 dst: /127.0.0.1:35263
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:56,153 WARN  [Thread-1083 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42413,DS-279227df-06e6-42dd-b2a7-b11135e8d57c,DISK]
2024-11-19T07:34:56,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741901_1086 (size=5424)
2024-11-19T07:34:56,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741901_1086 (size=5424)
2024-11-19T07:34:56,158 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/table/097a65b540ad447281b0f215e2832207
2024-11-19T07:34:56,167 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/info/1f3348e89d544bb38b8d8e8f8560fbb2 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/info/1f3348e89d544bb38b8d8e8f8560fbb2
2024-11-19T07:34:56,175 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/info/1f3348e89d544bb38b8d8e8f8560fbb2, entries=10, sequenceid=11, filesize=6.9 K
2024-11-19T07:34:56,177 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/ns/f2cbbb3e1d4a4d03ae2ced267aa5900b as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/ns/f2cbbb3e1d4a4d03ae2ced267aa5900b
2024-11-19T07:34:56,185 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/ns/f2cbbb3e1d4a4d03ae2ced267aa5900b, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T07:34:56,187 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/.tmp/table/097a65b540ad447281b0f215e2832207 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/table/097a65b540ad447281b0f215e2832207
2024-11-19T07:34:56,195 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/table/097a65b540ad447281b0f215e2832207, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T07:34:56,196 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 154ms, sequenceid=11, compaction requested=false
2024-11-19T07:34:56,205 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T07:34:56,206 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:34:56,206 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:56,206 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001696042Running coprocessor pre-close hooks at 1732001696042Disabling compacts and flushes for region at 1732001696042Disabling writes for close at 1732001696042Obtaining lock to block concurrent updates at 1732001696042Preparing flush snapshotting stores in 1588230740 at 1732001696042Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732001696043 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732001696058 (+15 ms)Flushing 1588230740/info: creating writer at 1732001696059 (+1 ms)Flushing 1588230740/info: appending metadata at 1732001696081 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732001696081Flushing 1588230740/ns: creating writer at 1732001696101 (+20 ms)Flushing 1588230740/ns: appending metadata at 1732001696118 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732001696118Flushing 1588230740/table: creating writer at 1732001696131 (+13 ms)Flushing 1588230740/table: appending metadata at 1732001696149 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732001696149Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba4370f: reopening flushed file at 1732001696166 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba1dea2: reopening flushed file at 1732001696176 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b4af91f: reopening flushed file at 1732001696186 (+10 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 154ms, sequenceid=11, compaction requested=false at 1732001696196 (+10 ms)Writing region close event to WAL at 1732001696201 (+5 ms)Running coprocessor post-close hooks at 1732001696205 (+4 ms)Closed at 1732001696206 (+1 ms)
2024-11-19T07:34:56,206 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:34:56,242 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,42239,1732001656169; all regions closed.
2024-11-19T07:34:56,242 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,37853,1732001657436; all regions closed.
2024-11-19T07:34:56,242 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,242 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,243 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:34:56,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741888_1072 (size=18156)
2024-11-19T07:34:56,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741888_1072 (size=18156)
2024-11-19T07:34:56,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741895_1079 (size=825)
2024-11-19T07:34:56,248 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741895_1079 (size=825)
2024-11-19T07:34:56,250 DEBUG [RS:1;db0afee3eab9:37853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs
2024-11-19T07:34:56,250 INFO  [RS:1;db0afee3eab9:37853 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C37853%2C1732001657436:(num 1732001695322)
2024-11-19T07:34:56,250 DEBUG [RS:1;db0afee3eab9:37853 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:34:56,250 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:56,250 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:34:56,251 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:34:56,251 INFO  [RS:1;db0afee3eab9:37853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37853
2024-11-19T07:34:56,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:34:56,416 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,37853,1732001657436
2024-11-19T07:34:56,416 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:34:56,416 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,37853,1732001657436]
2024-11-19T07:34:56,556 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,37853,1732001657436 already deleted, retry=false
2024-11-19T07:34:56,556 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,37853,1732001657436 expired; onlineServers=1
2024-11-19T07:34:56,600 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:56,600 INFO  [RS:1;db0afee3eab9:37853 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:34:56,600 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37853-0x1015203e85d0002, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:34:56,600 INFO  [RS:1;db0afee3eab9:37853 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,37853,1732001657436; zookeeper connection closed.
2024-11-19T07:34:56,601 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7da6fd3f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7da6fd3f
2024-11-19T07:34:56,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741875_1058 (size=12911)
2024-11-19T07:34:56,669 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:56,747 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T07:34:56,747 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T07:34:57,540 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:34:57,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:34:57,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741836_1012 (size=76)
2024-11-19T07:34:57,947 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T07:34:57,947 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T07:34:58,058 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:34:58,671 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3ac44c4d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35263, datanodeUuid=4c0ab2de-fd04-4658-ad1b-d728f7cc4f29, infoPort=39479, infoSecurePort=0, ipcPort=36413, storageInfo=lv=-57;cid=testClusterID;nsid=850737210;c=1732001653232):Failed to transfer BP-1194886781-172.17.0.2-1732001653232:blk_1073741828_1004 to 127.0.0.1:42413 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:34:58,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:35:00,058 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:00,059 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta after 4002ms
2024-11-19T07:35:01,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-19T07:35:01,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:35:01,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:35:01,105 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,147 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,244 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-19T07:35:01,248 DEBUG [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs
2024-11-19T07:35:01,248 INFO  [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C42239%2C1732001656169.meta:.meta(num 1732001696043)
2024-11-19T07:35:01,249 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:01,249 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:01,249 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:01,249 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:01,249 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:01,250 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:01,250 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44027,DS-66c39e22-4de6-4e40-8632-497e67c5d05c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:01,250 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
2024-11-19T07:35:01,251 WARN  [IPC Server handler 2 on default port 40659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 has not been closed. Lease recovery is in progress. RecoveryId = 1087 for block blk_1073741833_1009
2024-11-19T07:35:01,251 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 after 1ms
2024-11-19T07:35:01,649 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:01,671 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,672 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,672 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,673 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,673 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,674 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:01,682 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:02,059 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T07:35:02,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:02,691 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@358997b5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1194886781-172.17.0.2-1732001653232:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:44027,null,null])
java.net.ConnectException: Call From db0afee3eab9/172.17.0.2 to localhost:39579 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T07:35:03,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:04,059 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T07:35:04,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:05,072 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:05,253 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 after 4002ms
2024-11-19T07:35:05,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741830_1070 (size=27318)
2024-11-19T07:35:06,059 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-19T07:35:06,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:06,250 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-19T07:35:06,252 DEBUG [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/oldWALs
2024-11-19T07:35:06,252 INFO  [RS:0;db0afee3eab9:42239 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C42239%2C1732001656169:(num 1732001656802)
2024-11-19T07:35:06,252 DEBUG [RS:0;db0afee3eab9:42239 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:06,252 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:35:06,252 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:35:06,252 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:35:06,252 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:35:06,252 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:35:06,253 INFO  [RS:0;db0afee3eab9:42239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42239
2024-11-19T07:35:06,254 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:06,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,42239,1732001656169
2024-11-19T07:35:06,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:35:06,323 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:35:06,333 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,42239,1732001656169]
2024-11-19T07:35:06,343 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,42239,1732001656169 already deleted, retry=false
2024-11-19T07:35:06,343 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,42239,1732001656169 expired; onlineServers=0
2024-11-19T07:35:06,344 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,42985,1732001656005' *****
2024-11-19T07:35:06,344 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:35:06,344 INFO  [M:0;db0afee3eab9:42985 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:35:06,344 INFO  [M:0;db0afee3eab9:42985 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:35:06,344 DEBUG [M:0;db0afee3eab9:42985 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:35:06,344 DEBUG [M:0;db0afee3eab9:42985 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:35:06,344 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001656527 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001656527,5,FailOnTimeoutGroup]
2024-11-19T07:35:06,344 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001656531 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001656531,5,FailOnTimeoutGroup]
2024-11-19T07:35:06,344 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:35:06,344 INFO  [M:0;db0afee3eab9:42985 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:35:06,345 INFO  [M:0;db0afee3eab9:42985 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:35:06,345 DEBUG [M:0;db0afee3eab9:42985 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:35:06,345 INFO  [M:0;db0afee3eab9:42985 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:35:06,345 INFO  [M:0;db0afee3eab9:42985 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:35:06,345 INFO  [M:0;db0afee3eab9:42985 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:35:06,345 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:35:06,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:35:06,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:06,354 DEBUG [M:0;db0afee3eab9:42985 {}] zookeeper.ZKUtil(347): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:35:06,354 WARN  [M:0;db0afee3eab9:42985 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:35:06,355 INFO  [M:0;db0afee3eab9:42985 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/.lastflushedseqids
2024-11-19T07:35:06,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741902_1088 (size=130)
2024-11-19T07:35:06,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741902_1088 (size=130)
2024-11-19T07:35:06,364 INFO  [M:0;db0afee3eab9:42985 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:35:06,364 INFO  [M:0;db0afee3eab9:42985 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:35:06,364 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:35:06,364 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:06,364 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:06,364 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:35:06,364 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:06,365 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-19T07:35:06,384 DEBUG [M:0;db0afee3eab9:42985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e848421a66194458af15c8a53e3c951b is 82, key is hbase:meta,,1/info:regioninfo/1732001657194/Put/seqid=0
2024-11-19T07:35:06,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741903_1089 (size=5672)
2024-11-19T07:35:06,389 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e848421a66194458af15c8a53e3c951b
2024-11-19T07:35:06,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741903_1089 (size=5672)
2024-11-19T07:35:06,413 DEBUG [M:0;db0afee3eab9:42985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eef6ce83e508407688a9c6cc3355ceb3 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732001658374/Put/seqid=0
2024-11-19T07:35:06,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741904_1090 (size=6256)
2024-11-19T07:35:06,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741904_1090 (size=6256)
2024-11-19T07:35:06,423 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eef6ce83e508407688a9c6cc3355ceb3
2024-11-19T07:35:06,429 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eef6ce83e508407688a9c6cc3355ceb3
2024-11-19T07:35:06,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:06,433 INFO  [RS:0;db0afee3eab9:42239 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:35:06,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42239-0x1015203e85d0001, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:06,433 INFO  [RS:0;db0afee3eab9:42239 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,42239,1732001656169; zookeeper connection closed.
2024-11-19T07:35:06,433 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29b13f65 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29b13f65
2024-11-19T07:35:06,433 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-19T07:35:06,450 DEBUG [M:0;db0afee3eab9:42985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b477afbf6ec4c3484baaf059accd437 is 69, key is db0afee3eab9,37853,1732001657436/rs:state/1732001657511/Put/seqid=0
2024-11-19T07:35:06,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741905_1091 (size=5224)
2024-11-19T07:35:06,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741905_1091 (size=5224)
2024-11-19T07:35:06,456 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b477afbf6ec4c3484baaf059accd437
2024-11-19T07:35:06,475 DEBUG [M:0;db0afee3eab9:42985 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74448ecebded41d69767e40be80b29a0 is 52, key is load_balancer_on/state:d/1732001657415/Put/seqid=0
2024-11-19T07:35:06,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741906_1092 (size=5056)
2024-11-19T07:35:06,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741906_1092 (size=5056)
2024-11-19T07:35:06,481 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74448ecebded41d69767e40be80b29a0
2024-11-19T07:35:06,488 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e848421a66194458af15c8a53e3c951b as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e848421a66194458af15c8a53e3c951b
2024-11-19T07:35:06,495 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e848421a66194458af15c8a53e3c951b, entries=8, sequenceid=60, filesize=5.5 K
2024-11-19T07:35:06,496 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eef6ce83e508407688a9c6cc3355ceb3 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eef6ce83e508407688a9c6cc3355ceb3
2024-11-19T07:35:06,502 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eef6ce83e508407688a9c6cc3355ceb3
2024-11-19T07:35:06,502 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eef6ce83e508407688a9c6cc3355ceb3, entries=6, sequenceid=60, filesize=6.1 K
2024-11-19T07:35:06,507 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b477afbf6ec4c3484baaf059accd437 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b477afbf6ec4c3484baaf059accd437
2024-11-19T07:35:06,513 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b477afbf6ec4c3484baaf059accd437, entries=2, sequenceid=60, filesize=5.1 K
2024-11-19T07:35:06,515 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74448ecebded41d69767e40be80b29a0 as hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/74448ecebded41d69767e40be80b29a0
2024-11-19T07:35:06,521 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/74448ecebded41d69767e40be80b29a0, entries=1, sequenceid=60, filesize=4.9 K
2024-11-19T07:35:06,522 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=60, compaction requested=false
2024-11-19T07:35:06,524 INFO  [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:06,524 DEBUG [M:0;db0afee3eab9:42985 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001706364Disabling compacts and flushes for region at 1732001706364Disabling writes for close at 1732001706364Obtaining lock to block concurrent updates at 1732001706365 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001706365Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1732001706365Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001706366 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001706366Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001706383 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001706383Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001706398 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001706412 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001706412Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001706429 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001706450 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001706450Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001706461 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001706474 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001706474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1172aea9: reopening flushed file at 1732001706487 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cbbc7bf: reopening flushed file at 1732001706495 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f83e92: reopening flushed file at 1732001706503 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54423211: reopening flushed file at 1732001706514 (+11 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=60, compaction requested=false at 1732001706522 (+8 ms)Writing region close event to WAL at 1732001706524 (+2 ms)Closed at 1732001706524
2024-11-19T07:35:06,524 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:06,525 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:06,525 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:06,525 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:06,525 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:06,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39757 is added to blk_1073741886_1069 (size=1045)
2024-11-19T07:35:06,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35263 is added to blk_1073741886_1069 (size=1045)
2024-11-19T07:35:06,581 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:06,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,597 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,601 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,604 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:06,928 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:35:06,928 INFO  [M:0;db0afee3eab9:42985 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:35:06,929 INFO  [M:0;db0afee3eab9:42985 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42985
2024-11-19T07:35:06,929 INFO  [M:0;db0afee3eab9:42985 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:35:07,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:07,044 INFO  [M:0;db0afee3eab9:42985 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:35:07,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42985-0x1015203e85d0000, quorum=127.0.0.1:62643, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:07,047 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@122a196d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:07,047 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a528add{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:07,047 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:07,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a8eeeb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:07,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d6a3337{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:07,049 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:07,049 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:07,049 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:07,049 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 4c0ab2de-fd04-4658-ad1b-d728f7cc4f29) service to localhost/127.0.0.1:40659
2024-11-19T07:35:07,049 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@ad9dce8 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:44027,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:39579 , LocalHost:localPort db0afee3eab9/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T07:35:07,050 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@ad9dce8 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1194886781-172.17.0.2-1732001653232:blk_1073741833_1009; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:35263,null,null], DatanodeInfoWithStorage[127.0.0.1:44027,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1194886781-172.17.0.2-1732001653232
2024-11-19T07:35:07,051 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data3/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:07,051 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6baa5a22 {}] ipc.Client$Connection(956): Interrupted while trying for connection
2024-11-19T07:35:07,051 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data4/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:07,052 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:07,051 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6baa5a22 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:44027,null,null])
java.net.ConnectException: Call From db0afee3eab9/172.17.0.2 to localhost:39579 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-19T07:35:07,052 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6baa5a22 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1194886781-172.17.0.2-1732001653232:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:35263,null,null], DatanodeInfoWithStorage[127.0.0.1:44027,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1194886781-172.17.0.2-1732001653232
2024-11-19T07:35:07,055 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49584ab0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:07,055 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59d0863f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:07,056 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:07,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3eb2cdd3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:07,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ed35b1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:07,061 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:07,061 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:07,061 WARN  [BP-1194886781-172.17.0.2-1732001653232 heartbeating to localhost/127.0.0.1:40659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194886781-172.17.0.2-1732001653232 (Datanode Uuid 70018873-12a4-426e-bade-060972dd72d8) service to localhost/127.0.0.1:40659
2024-11-19T07:35:07,061 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:07,062 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data7/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:07,062 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/cluster_5e0b456e-85c8-cf5c-31cd-e3c6efbd1de7/data/data8/current/BP-1194886781-172.17.0.2-1732001653232 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:07,063 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:07,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2606b08f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:35:07,068 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:07,068 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:07,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:07,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:07,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:07,076 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:35:07,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:35:07,122 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78)
Potentially hanging thread: LeaseRenewer:jenkins@localhost:40659
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f768cbf5378.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40659
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40659
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40659 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42993
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40659
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40659
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40659
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f768cbf5378.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:42993
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40659 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40659 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=429 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 156) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11582 (was 12504)
2024-11-19T07:35:07,129 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=429, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=11582
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.log.dir so I do NOT create it in target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cbaa57b-5c3c-d696-7cd4-a53b053fcac6/hadoop.tmp.dir so I do NOT create it in target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da, deleteOnExit=true
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/test.cache.data in system properties and HBase conf
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:35:07,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:35:07,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:35:07,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:35:07,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:35:07,146 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:35:07,255 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:07,508 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:07,514 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:07,515 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:07,515 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:07,515 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:35:07,516 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:07,520 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7096145a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:07,521 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10c583a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:07,622 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@232fa1ae{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-44863-hadoop-hdfs-3_4_1-tests_jar-_-any-11129797939794888931/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:35:07,622 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c07fc0{HTTP/1.1, (http/1.1)}{localhost:44863}
2024-11-19T07:35:07,623 INFO  [Time-limited test {}] server.Server(415): Started @164311ms
2024-11-19T07:35:07,636 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:35:07,871 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:07,875 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:07,876 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:07,876 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:07,876 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:07,876 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13646a74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:07,877 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@301612e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:07,982 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44e11af1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-40523-hadoop-hdfs-3_4_1-tests_jar-_-any-2316233946743170479/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:07,982 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60043bb8{HTTP/1.1, (http/1.1)}{localhost:40523}
2024-11-19T07:35:07,983 INFO  [Time-limited test {}] server.Server(415): Started @164672ms
2024-11-19T07:35:07,984 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:08,021 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:08,025 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:08,026 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:08,026 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:08,026 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:08,027 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47946b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:08,028 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23accf28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:08,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:08,155 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77c83f30{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-40897-hadoop-hdfs-3_4_1-tests_jar-_-any-5237556126932774071/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:08,156 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54dbaae8{HTTP/1.1, (http/1.1)}{localhost:40897}
2024-11-19T07:35:08,156 INFO  [Time-limited test {}] server.Server(415): Started @164845ms
2024-11-19T07:35:08,157 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:08,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:08,970 WARN  [Thread-1223 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data1/current/BP-1016362256-172.17.0.2-1732001707159/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:08,970 WARN  [Thread-1224 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data2/current/BP-1016362256-172.17.0.2-1732001707159/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:08,986 WARN  [Thread-1187 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:08,988 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a4e692de08a6233 with lease ID 0x1c8c85c88abc22a3: Processing first storage report for DS-c5074078-4407-43da-8052-4cc1948c719f from datanode DatanodeRegistration(127.0.0.1:35141, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=44661, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159)
2024-11-19T07:35:08,989 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a4e692de08a6233 with lease ID 0x1c8c85c88abc22a3: from storage DS-c5074078-4407-43da-8052-4cc1948c719f node DatanodeRegistration(127.0.0.1:35141, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=44661, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:08,989 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a4e692de08a6233 with lease ID 0x1c8c85c88abc22a3: Processing first storage report for DS-3f57fe33-994d-4cf7-9ba1-b243d02570b5 from datanode DatanodeRegistration(127.0.0.1:35141, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=44661, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159)
2024-11-19T07:35:08,989 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a4e692de08a6233 with lease ID 0x1c8c85c88abc22a3: from storage DS-3f57fe33-994d-4cf7-9ba1-b243d02570b5 node DatanodeRegistration(127.0.0.1:35141, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=44661, infoSecurePort=0, ipcPort=41861, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:09,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:09,162 WARN  [Thread-1235 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data4/current/BP-1016362256-172.17.0.2-1732001707159/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:09,162 WARN  [Thread-1234 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data3/current/BP-1016362256-172.17.0.2-1732001707159/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:09,183 WARN  [Thread-1210 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:09,186 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80ddc4224dabb8df with lease ID 0x1c8c85c88abc22a4: Processing first storage report for DS-fd076b65-1660-424c-a802-4a7f1eb23f56 from datanode DatanodeRegistration(127.0.0.1:35391, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=33473, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159)
2024-11-19T07:35:09,186 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80ddc4224dabb8df with lease ID 0x1c8c85c88abc22a4: from storage DS-fd076b65-1660-424c-a802-4a7f1eb23f56 node DatanodeRegistration(127.0.0.1:35391, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=33473, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T07:35:09,186 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80ddc4224dabb8df with lease ID 0x1c8c85c88abc22a4: Processing first storage report for DS-93eea236-07fd-4061-905b-21b9f469272c from datanode DatanodeRegistration(127.0.0.1:35391, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=33473, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159)
2024-11-19T07:35:09,186 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80ddc4224dabb8df with lease ID 0x1c8c85c88abc22a4: from storage DS-93eea236-07fd-4061-905b-21b9f469272c node DatanodeRegistration(127.0.0.1:35391, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=33473, infoSecurePort=0, ipcPort=37935, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:09,198 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32
2024-11-19T07:35:09,202 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/zookeeper_0, clientPort=53294, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:35:09,202 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53294
2024-11-19T07:35:09,203 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,204 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:35:09,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:35:09,215 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b with version=8
2024-11-19T07:35:09,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:35:09,217 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:35:09,217 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,217 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,217 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:35:09,217 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,218 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:35:09,218 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:35:09,218 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:35:09,218 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41197
2024-11-19T07:35:09,220 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41197 connecting to ZooKeeper ensemble=127.0.0.1:53294
2024-11-19T07:35:09,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:09,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:411970x0, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:35:09,277 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41197-0x1015204b83b0000 connected
2024-11-19T07:35:09,354 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,356 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,358 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:09,359 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b, hbase.cluster.distributed=false
2024-11-19T07:35:09,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:35:09,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41197
2024-11-19T07:35:09,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41197
2024-11-19T07:35:09,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41197
2024-11-19T07:35:09,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41197
2024-11-19T07:35:09,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41197
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:09,382 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:35:09,383 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:35:09,383 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:35:09,383 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38857
2024-11-19T07:35:09,385 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38857 connecting to ZooKeeper ensemble=127.0.0.1:53294
2024-11-19T07:35:09,386 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,388 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388570x0, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:35:09,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388570x0, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:09,396 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38857-0x1015204b83b0001 connected
2024-11-19T07:35:09,397 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:35:09,397 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:35:09,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:35:09,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:35:09,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38857
2024-11-19T07:35:09,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38857
2024-11-19T07:35:09,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38857
2024-11-19T07:35:09,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38857
2024-11-19T07:35:09,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38857
2024-11-19T07:35:09,433 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:41197
2024-11-19T07:35:09,433 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:09,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:09,438 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:35:09,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,449 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:35:09,449 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,41197,1732001709217 from backup master directory
2024-11-19T07:35:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:09,459 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:35:09,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:09,459 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,464 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/hbase.id] with ID: 58489e42-f70f-443b-a499-f61ee1dfef27
2024-11-19T07:35:09,464 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/.tmp/hbase.id
2024-11-19T07:35:09,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:35:09,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:35:09,470 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/.tmp/hbase.id]:[hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/hbase.id]
2024-11-19T07:35:09,485 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:09,485 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:35:09,486 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-19T07:35:09,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:35:09,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:35:09,508 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:35:09,509 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:35:09,509 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:09,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:35:09,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:35:09,517 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store
2024-11-19T07:35:09,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:35:09,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:35:09,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:09,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:35:09,524 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:09,524 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:09,525 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:35:09,525 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:09,525 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:09,525 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001709524Disabling compacts and flushes for region at 1732001709524Disabling writes for close at 1732001709525 (+1 ms)Writing region close event to WAL at 1732001709525Closed at 1732001709525
2024-11-19T07:35:09,526 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/.initializing
2024-11-19T07:35:09,526 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,528 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C41197%2C1732001709217, suffix=, logDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217, archiveDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/oldWALs, maxLogs=10
2024-11-19T07:35:09,529 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C41197%2C1732001709217.1732001709529
2024-11-19T07:35:09,534 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529
2024-11-19T07:35:09,534 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44661:44661),(127.0.0.1/127.0.0.1:33473:33473)]
2024-11-19T07:35:09,535 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:09,535 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:09,535 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,535 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,540 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:09,542 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:35:09,542 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:09,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:09,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:09,544 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:35:09,544 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:09,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:09,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:09,546 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:35:09,546 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:09,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:09,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:09,548 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:35:09,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:09,549 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:09,549 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,549 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,550 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,551 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,551 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,552 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:35:09,553 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:09,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:09,556 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=803310, jitterRate=0.02146221697330475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:35:09,556 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001709535Initializing all the Stores at 1732001709536 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001709536Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001709540 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001709540Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001709540Cleaning up temporary data from old regions at 1732001709551 (+11 ms)Region opened successfully at 1732001709556 (+5 ms)
2024-11-19T07:35:09,560 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:35:09,564 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e9e7bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:35:09,565 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:35:09,565 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:35:09,565 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:35:09,566 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:35:09,566 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:35:09,567 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:35:09,567 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:35:09,569 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:35:09,570 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:35:09,662 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:35:09,662 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:35:09,663 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:35:09,710 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:35:09,711 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:35:09,712 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:35:09,722 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:35:09,724 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:35:09,733 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:35:09,735 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:35:09,743 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:35:09,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:09,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:09,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,755 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,41197,1732001709217, sessionid=0x1015204b83b0000, setting cluster-up flag (Was=false)
2024-11-19T07:35:09,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,806 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:35:09,807 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:09,859 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:35:09,860 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,41197,1732001709217
2024-11-19T07:35:09,862 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:35:09,863 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:09,864 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:35:09,864 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:35:09,864 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,41197,1732001709217
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:35:09,865 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:09,865 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:09,865 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:09,865 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:09,865 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:35:09,866 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,866 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:35:09,866 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,866 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001739866
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:09,867 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:35:09,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:35:09,868 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001709868,5,FailOnTimeoutGroup]
2024-11-19T07:35:09,868 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001709868,5,FailOnTimeoutGroup]
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,868 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:09,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,868 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:35:09,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:35:09,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:35:09,922 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(746): ClusterId : 58489e42-f70f-443b-a499-f61ee1dfef27
2024-11-19T07:35:09,923 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:35:09,934 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:35:09,934 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:35:09,944 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:35:09,945 DEBUG [RS:0;db0afee3eab9:38857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5630a8de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:35:09,959 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:38857
2024-11-19T07:35:09,959 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:35:09,959 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:35:09,959 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:35:09,960 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,41197,1732001709217 with port=38857, startcode=1732001709382
2024-11-19T07:35:09,960 DEBUG [RS:0;db0afee3eab9:38857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:35:09,962 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:35:09,962 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41197 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,38857,1732001709382
2024-11-19T07:35:09,962 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41197 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:09,964 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b
2024-11-19T07:35:09,964 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34215
2024-11-19T07:35:09,964 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:35:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:35:09,975 DEBUG [RS:0;db0afee3eab9:38857 {}] zookeeper.ZKUtil(111): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,38857,1732001709382
2024-11-19T07:35:09,975 WARN  [RS:0;db0afee3eab9:38857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:35:09,975 INFO  [RS:0;db0afee3eab9:38857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:09,975 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382
2024-11-19T07:35:09,976 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,38857,1732001709382]
2024-11-19T07:35:09,979 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:35:09,980 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:35:09,981 INFO  [RS:0;db0afee3eab9:38857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:35:09,981 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,981 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:35:09,982 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:35:09,982 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:35:09,982 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:35:09,983 DEBUG [RS:0;db0afee3eab9:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:35:09,988 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,988 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,989 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,989 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,989 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:09,989 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,38857,1732001709382-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:35:10,005 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:35:10,005 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,38857,1732001709382-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,005 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,005 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.Replication(171): db0afee3eab9,38857,1732001709382 started
2024-11-19T07:35:10,018 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,019 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,38857,1732001709382, RpcServer on db0afee3eab9/172.17.0.2:38857, sessionid=0x1015204b83b0001
2024-11-19T07:35:10,019 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:35:10,019 DEBUG [RS:0;db0afee3eab9:38857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,38857,1732001709382
2024-11-19T07:35:10,019 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,38857,1732001709382'
2024-11-19T07:35:10,019 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:35:10,019 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,38857,1732001709382
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,38857,1732001709382'
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:35:10,020 DEBUG [RS:0;db0afee3eab9:38857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:35:10,020 INFO  [RS:0;db0afee3eab9:38857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:35:10,020 INFO  [RS:0;db0afee3eab9:38857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:35:10,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:10,123 INFO  [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C38857%2C1732001709382, suffix=, logDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382, archiveDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs, maxLogs=32
2024-11-19T07:35:10,124 INFO  [RS:0;db0afee3eab9:38857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:10,135 INFO  [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:10,136 DEBUG [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33473:33473),(127.0.0.1/127.0.0.1:44661:44661)]
2024-11-19T07:35:10,257 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:10,276 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:35:10,276 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b
2024-11-19T07:35:10,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741833_1009 (size=32)
2024-11-19T07:35:10,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741833_1009 (size=32)
2024-11-19T07:35:10,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:10,288 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:35:10,290 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:35:10,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,290 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,290 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:35:10,292 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:35:10,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,293 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,293 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:35:10,294 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:35:10,294 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,295 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,295 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:35:10,296 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:35:10,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,297 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:35:10,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740
2024-11-19T07:35:10,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740
2024-11-19T07:35:10,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:35:10,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:35:10,300 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:35:10,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:35:10,303 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:10,303 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=764199, jitterRate=-0.028270870447158813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:35:10,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001710287Initializing all the Stores at 1732001710288 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710288Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710288Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001710288Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710288Cleaning up temporary data from old regions at 1732001710299 (+11 ms)Region opened successfully at 1732001710304 (+5 ms)
2024-11-19T07:35:10,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:35:10,304 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:35:10,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:35:10,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:35:10,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:35:10,305 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:35:10,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001710304Disabling compacts and flushes for region at 1732001710304Disabling writes for close at 1732001710304Writing region close event to WAL at 1732001710305 (+1 ms)Closed at 1732001710305
2024-11-19T07:35:10,306 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:10,306 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:35:10,306 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:35:10,307 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:35:10,309 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:35:10,459 DEBUG [db0afee3eab9:41197 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:35:10,460 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:10,462 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,38857,1732001709382, state=OPENING
2024-11-19T07:35:10,512 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:35:10,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:10,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:10,523 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:35:10,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:10,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:10,523 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,38857,1732001709382}]
2024-11-19T07:35:10,677 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:35:10,679 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33231, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:35:10,682 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:35:10,683 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:10,684 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C38857%2C1732001709382.meta, suffix=.meta, logDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382, archiveDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs, maxLogs=32
2024-11-19T07:35:10,685 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta
2024-11-19T07:35:10,690 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta
2024-11-19T07:35:10,691 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33473:33473),(127.0.0.1/127.0.0.1:44661:44661)]
2024-11-19T07:35:10,691 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:10,691 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:35:10,691 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:35:10,692 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:35:10,692 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:35:10,692 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:10,692 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:35:10,692 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:35:10,696 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:35:10,697 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:35:10,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,698 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,698 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:35:10,699 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:35:10,699 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,700 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,700 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:35:10,700 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:35:10,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,701 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,701 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:35:10,702 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:35:10,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,702 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:10,702 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:35:10,703 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740
2024-11-19T07:35:10,704 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740
2024-11-19T07:35:10,705 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:35:10,705 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:35:10,706 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:35:10,707 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:35:10,708 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826567, jitterRate=0.05103525519371033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:35:10,708 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:35:10,709 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001710692Writing region info on filesystem at 1732001710692Initializing all the Stores at 1732001710693 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710693Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710696 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001710696Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001710696Cleaning up temporary data from old regions at 1732001710705 (+9 ms)Running coprocessor post-open hooks at 1732001710708 (+3 ms)Region opened successfully at 1732001710708
2024-11-19T07:35:10,709 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001710677
2024-11-19T07:35:10,712 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:35:10,712 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:35:10,713 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:10,714 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,38857,1732001709382, state=OPEN
2024-11-19T07:35:10,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:35:10,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:35:10,750 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:10,750 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:10,750 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:10,754 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:35:10,754 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,38857,1732001709382 in 227 msec
2024-11-19T07:35:10,757 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:35:10,757 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 448 msec
2024-11-19T07:35:10,758 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:10,758 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:35:10,759 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:35:10,759 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,38857,1732001709382, seqNum=-1]
2024-11-19T07:35:10,760 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:35:10,761 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:35:10,767 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 903 msec
2024-11-19T07:35:10,767 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001710767, completionTime=-1
2024-11-19T07:35:10,767 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:35:10,767 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:35:10,769 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001770769
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001830770
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:41197, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,770 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,771 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,772 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:35:10,775 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.316sec
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:35:10,776 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:35:10,779 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:35:10,779 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:35:10,779 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,41197,1732001709217-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:10,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370b05c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:10,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,41197,-1 for getting cluster id
2024-11-19T07:35:10,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:35:10,824 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '58489e42-f70f-443b-a499-f61ee1dfef27'
2024-11-19T07:35:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:35:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "58489e42-f70f-443b-a499-f61ee1dfef27"

2024-11-19T07:35:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651aa95f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:10,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,41197,-1]
2024-11-19T07:35:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:35:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:10,827 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:35:10,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b48cd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:10,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:35:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,38857,1732001709382, seqNum=-1]
2024-11-19T07:35:10,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:35:10,834 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:35:10,836 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,41197,1732001709217
2024-11-19T07:35:10,836 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:10,840 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:35:10,840 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-19T07:35:10,840 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-19T07:35:10,840 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T07:35:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,41197,1732001709217
2024-11-19T07:35:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42c4f0fe
2024-11-19T07:35:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T07:35:10,844 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T07:35:10,845 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T07:35:10,845 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T07:35:10,845 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:35:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T07:35:10,848 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T07:35:10,849 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:10,849 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-19T07:35:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:35:10,850 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T07:35:10,857 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741835_1011 (size=395)
2024-11-19T07:35:10,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741835_1011 (size=395)
2024-11-19T07:35:10,859 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94dbd9302ccd7473466748db5c92e1c9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b
2024-11-19T07:35:10,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35391 is added to blk_1073741836_1012 (size=78)
2024-11-19T07:35:10,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35141 is added to blk_1073741836_1012 (size=78)
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 94dbd9302ccd7473466748db5c92e1c9, disabling compactions & flushes
2024-11-19T07:35:10,866 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9. after waiting 0 ms
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:10,866 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:10,866 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94dbd9302ccd7473466748db5c92e1c9:
Waiting for close lock at 1732001710866Disabling compacts and flushes for region at 1732001710866Disabling writes for close at 1732001710866Writing region close event to WAL at 1732001710866Closed at 1732001710866
2024-11-19T07:35:10,868 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T07:35:10,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732001710868"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001710868"}]},"ts":"1732001710868"}
2024-11-19T07:35:10,870 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T07:35:10,872 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T07:35:10,872 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001710872"}]},"ts":"1732001710872"}
2024-11-19T07:35:10,874 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-19T07:35:10,875 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=94dbd9302ccd7473466748db5c92e1c9, ASSIGN}]
2024-11-19T07:35:10,876 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=94dbd9302ccd7473466748db5c92e1c9, ASSIGN
2024-11-19T07:35:10,877 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=94dbd9302ccd7473466748db5c92e1c9, ASSIGN; state=OFFLINE, location=db0afee3eab9,38857,1732001709382; forceNewPlan=false, retain=false
2024-11-19T07:35:11,028 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94dbd9302ccd7473466748db5c92e1c9, regionState=OPENING, regionLocation=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:11,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=94dbd9302ccd7473466748db5c92e1c9, ASSIGN because future has completed
2024-11-19T07:35:11,032 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94dbd9302ccd7473466748db5c92e1c9, server=db0afee3eab9,38857,1732001709382}]
2024-11-19T07:35:11,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:35:11,077 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T07:35:11,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:11,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:35:11,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T07:35:11,190 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:11,190 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 94dbd9302ccd7473466748db5c92e1c9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:11,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:11,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,191 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,192 INFO  [StoreOpener-94dbd9302ccd7473466748db5c92e1c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 94dbd9302ccd7473466748db5c92e1c9 
2024-11-19T07:35:11,194 INFO  [StoreOpener-94dbd9302ccd7473466748db5c92e1c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94dbd9302ccd7473466748db5c92e1c9 columnFamilyName info
2024-11-19T07:35:11,194 DEBUG [StoreOpener-94dbd9302ccd7473466748db5c92e1c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:11,194 INFO  [StoreOpener-94dbd9302ccd7473466748db5c92e1c9-1 {}] regionserver.HStore(327): Store=94dbd9302ccd7473466748db5c92e1c9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:11,195 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,195 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,196 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,196 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,196 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,198 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,203 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:11,204 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 94dbd9302ccd7473466748db5c92e1c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756190, jitterRate=-0.038454875349998474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:35:11,204 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:11,205 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 94dbd9302ccd7473466748db5c92e1c9:
Running coprocessor pre-open hook at 1732001711191Writing region info on filesystem at 1732001711191Initializing all the Stores at 1732001711192 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001711192Cleaning up temporary data from old regions at 1732001711196 (+4 ms)Running coprocessor post-open hooks at 1732001711204 (+8 ms)Region opened successfully at 1732001711205 (+1 ms)
2024-11-19T07:35:11,206 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9., pid=6, masterSystemTime=1732001711186
2024-11-19T07:35:11,209 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:11,209 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:11,210 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94dbd9302ccd7473466748db5c92e1c9, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,38857,1732001709382
2024-11-19T07:35:11,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94dbd9302ccd7473466748db5c92e1c9, server=db0afee3eab9,38857,1732001709382 because future has completed
2024-11-19T07:35:11,217 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T07:35:11,217 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 94dbd9302ccd7473466748db5c92e1c9, server=db0afee3eab9,38857,1732001709382 in 182 msec
2024-11-19T07:35:11,220 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T07:35:11,220 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=94dbd9302ccd7473466748db5c92e1c9, ASSIGN in 342 msec
2024-11-19T07:35:11,221 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T07:35:11,221 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001711221"}]},"ts":"1732001711221"}
2024-11-19T07:35:11,223 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-19T07:35:11,225 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T07:35:11,227 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 380 msec
2024-11-19T07:35:11,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:12,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:12,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:13,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:13,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:14,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:14,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:15,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:15,260 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:16,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:16,194 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:16,216 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,217 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,217 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,217 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,217 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,218 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,221 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,222 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,222 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,224 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:16,230 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:35:16,231 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-19T07:35:16,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:17,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:17,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:18,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:18,262 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:19,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:19,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:20,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:20,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41197 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:35:20,948 INFO  [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-19T07:35:20,948 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-19T07:35:20,951 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T07:35:20,951 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9., hostname=db0afee3eab9,38857,1732001709382, seqNum=2]
2024-11-19T07:35:21,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T07:35:21,077 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-19T07:35:21,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:21,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:22,086 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:22,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:22,958 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:22,958 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:22,959 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:22,959 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:22,959 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK], DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]) is bad.
2024-11-19T07:35:22,959 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK], DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]) is bad.
2024-11-19T07:35:22,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:47402 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35391:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47402 dst: /127.0.0.1:35391
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,960 WARN  [PacketResponder: BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35391] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:47376 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:35391:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47376 dst: /127.0.0.1:35391
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1323575647_22 at /127.0.0.1:42400 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42400 dst: /127.0.0.1:35141
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:42422 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42422 dst: /127.0.0.1:35141
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,959 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK], DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35391,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]) is bad.
2024-11-19T07:35:22,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1323575647_22 at /127.0.0.1:47366 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35391:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47366 dst: /127.0.0.1:35391
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:42410 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42410 dst: /127.0.0.1:35141
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:22,997 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77c83f30{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:22,997 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54dbaae8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:22,997 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:22,997 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23accf28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:22,997 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47946b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:22,998 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:22,998 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:22,999 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid c285a081-1271-4a07-b660-ba1b01062ae8) service to localhost/127.0.0.1:34215
2024-11-19T07:35:22,999 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:22,999 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data3/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:22,999 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data4/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:22,999 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:23,015 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:23,018 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:23,019 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:23,019 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:23,019 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:35:23,020 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75f58649{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:23,020 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@571de0fd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:23,087 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:23,124 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45604664{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-38433-hadoop-hdfs-3_4_1-tests_jar-_-any-17819815036710237053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:23,125 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e793ffb{HTTP/1.1, (http/1.1)}{localhost:38433}
2024-11-19T07:35:23,125 INFO  [Time-limited test {}] server.Server(415): Started @179814ms
2024-11-19T07:35:23,126 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:23,151 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:23,151 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:23,151 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:23,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:40932 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40932 dst: /127.0.0.1:35141
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:23,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:40934 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40934 dst: /127.0.0.1:35141
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:23,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1323575647_22 at /127.0.0.1:40954 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40954 dst: /127.0.0.1:35141
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:23,164 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44e11af1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:23,165 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60043bb8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:23,165 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:23,165 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@301612e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:23,165 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13646a74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:23,166 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:23,166 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid 50168649-4975-48fe-86c3-cb95fec33746) service to localhost/127.0.0.1:34215
2024-11-19T07:35:23,166 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data1/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:23,167 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data2/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:23,167 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:23,167 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:23,167 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:23,186 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:23,190 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:23,192 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:23,192 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:23,193 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:23,193 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ede944f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:23,193 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17312068{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:23,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:23,301 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ff0f915{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-33771-hadoop-hdfs-3_4_1-tests_jar-_-any-5492905140947966836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:23,301 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a9274b{HTTP/1.1, (http/1.1)}{localhost:33771}
2024-11-19T07:35:23,301 INFO  [Time-limited test {}] server.Server(415): Started @179990ms
2024-11-19T07:35:23,303 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:23,682 WARN  [Thread-1358 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:23,684 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x111b4347449d00dc with lease ID 0x1c8c85c88abc22a5: from storage DS-fd076b65-1660-424c-a802-4a7f1eb23f56 node DatanodeRegistration(127.0.0.1:39373, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=36191, infoSecurePort=0, ipcPort=33781, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:23,684 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x111b4347449d00dc with lease ID 0x1c8c85c88abc22a5: from storage DS-93eea236-07fd-4061-905b-21b9f469272c node DatanodeRegistration(127.0.0.1:39373, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=36191, infoSecurePort=0, ipcPort=33781, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:23,838 WARN  [Thread-1378 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:23,840 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd938575e390d6b55 with lease ID 0x1c8c85c88abc22a6: from storage DS-c5074078-4407-43da-8052-4cc1948c719f node DatanodeRegistration(127.0.0.1:37703, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=37637, infoSecurePort=0, ipcPort=42183, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:23,840 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd938575e390d6b55 with lease ID 0x1c8c85c88abc22a6: from storage DS-3f57fe33-994d-4cf7-9ba1-b243d02570b5 node DatanodeRegistration(127.0.0.1:37703, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=37637, infoSecurePort=0, ipcPort=42183, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:24,087 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:24,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:24,322 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-19T07:35:24,325 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-19T07:35:24,328 ERROR [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:24,328 WARN  [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:24,328 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C38857%2C1732001709382:(num 1732001710124) roll requested
2024-11-19T07:35:24,328 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:24,336 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 newFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:24,336 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:24,336 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:24,336 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:24,336 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:24,336 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:24,336 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:24,337 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:24,337 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:24,337 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:24,337 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37637:37637),(127.0.0.1/127.0.0.1:36191:36191)]
2024-11-19T07:35:24,338 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 is not closed yet, will try archiving it next time
2024-11-19T07:35:24,338 WARN  [IPC Server handler 3 on default port 34215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741832_1013
2024-11-19T07:35:24,338 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 after 1ms
2024-11-19T07:35:25,088 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:25,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:26,089 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:26,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:26,342 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-19T07:35:26,685 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741832_1013: GenerationStamp not matched, existing replica is blk_1073741832_1008
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T07:35:27,090 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:27,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:28,090 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:28,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:28,339 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 after 4002ms
2024-11-19T07:35:28,345 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:28,345 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37703,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK], DatanodeInfoWithStorage[127.0.0.1:39373,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37703,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]) is bad.
2024-11-19T07:35:28,346 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:44710 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39373:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44710 dst: /127.0.0.1:39373
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:28,346 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:33064 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37703:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33064 dst: /127.0.0.1:37703
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:28,375 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ff0f915{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:28,376 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a9274b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:28,376 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:28,376 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17312068{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:28,376 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ede944f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:28,378 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:28,378 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:28,378 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:28,378 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid 50168649-4975-48fe-86c3-cb95fec33746) service to localhost/127.0.0.1:34215
2024-11-19T07:35:28,379 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data1/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:28,379 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data2/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:28,379 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:28,389 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:28,399 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:28,400 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:28,400 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:28,400 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:28,401 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ef50a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:28,401 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6573e60c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:28,503 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a8b2822{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-44687-hadoop-hdfs-3_4_1-tests_jar-_-any-7426061336796752950/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:28,503 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68df0564{HTTP/1.1, (http/1.1)}{localhost:44687}
2024-11-19T07:35:28,503 INFO  [Time-limited test {}] server.Server(415): Started @185192ms
2024-11-19T07:35:28,504 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:28,541 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:28,541 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1702269614_22 at /127.0.0.1:44724 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39373:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44724 dst: /127.0.0.1:39373
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:28,543 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45604664{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:28,544 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e793ffb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:28,544 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:28,544 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@571de0fd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:28,544 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75f58649{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:28,545 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:28,545 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:28,545 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:28,545 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid c285a081-1271-4a07-b660-ba1b01062ae8) service to localhost/127.0.0.1:34215
2024-11-19T07:35:28,546 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data3/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:28,546 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data4/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:28,546 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:28,554 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:28,557 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:28,558 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:28,558 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:28,558 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:28,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f3e5a16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:28,559 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3420abff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:28,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c524311{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/java.io.tmpdir/jetty-localhost-44043-hadoop-hdfs-3_4_1-tests_jar-_-any-8958940246117754016/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:28,663 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@523d3401{HTTP/1.1, (http/1.1)}{localhost:44043}
2024-11-19T07:35:28,664 INFO  [Time-limited test {}] server.Server(415): Started @185352ms
2024-11-19T07:35:28,665 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:29,026 WARN  [Thread-1432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:29,028 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60cf71d0b4cc14ab with lease ID 0x1c8c85c88abc22a7: from storage DS-c5074078-4407-43da-8052-4cc1948c719f node DatanodeRegistration(127.0.0.1:36933, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=34413, infoSecurePort=0, ipcPort=45231, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:29,029 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60cf71d0b4cc14ab with lease ID 0x1c8c85c88abc22a7: from storage DS-3f57fe33-994d-4cf7-9ba1-b243d02570b5 node DatanodeRegistration(127.0.0.1:36933, datanodeUuid=50168649-4975-48fe-86c3-cb95fec33746, infoPort=34413, infoSecurePort=0, ipcPort=45231, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:29,091 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:29,186 WARN  [Thread-1452 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:29,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3d0aaec140decdd with lease ID 0x1c8c85c88abc22a8: from storage DS-fd076b65-1660-424c-a802-4a7f1eb23f56 node DatanodeRegistration(127.0.0.1:43771, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=36257, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:29,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3d0aaec140decdd with lease ID 0x1c8c85c88abc22a8: from storage DS-93eea236-07fd-4061-905b-21b9f469272c node DatanodeRegistration(127.0.0.1:43771, datanodeUuid=c285a081-1271-4a07-b660-ba1b01062ae8, infoPort=36257, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2134603072;c=1732001707159), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-19T07:35:29,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:29,685 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-19T07:35:29,688 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-19T07:35:29,690 ERROR [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39373,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:29,690 WARN  [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39373,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:29,690 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C38857%2C1732001709382:(num 1732001724328) roll requested
2024-11-19T07:35:29,690 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:29,698 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 newFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:29,698 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:29,698 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:29,698 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:29,698 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:29,698 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:29,699 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:29,699 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39373,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:29,699 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39373,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:29,699 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:29,699 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36257:36257),(127.0.0.1/127.0.0.1:34413:34413)]
2024-11-19T07:35:29,699 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 is not closed yet, will try archiving it next time
2024-11-19T07:35:29,699 WARN  [IPC Server handler 4 on default port 34215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-19T07:35:29,700 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 after 1ms
2024-11-19T07:35:30,092 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:30,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:31,092 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:31,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:31,701 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:31,706 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690 newFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:31,706 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:31,707 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:31,707 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:31,707 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:31,707 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:31,707 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:31,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34413:34413),(127.0.0.1/127.0.0.1:36257:36257)]
2024-11-19T07:35:31,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 is not closed yet, will try archiving it next time
2024-11-19T07:35:31,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690 is not closed yet, will try archiving it next time
2024-11-19T07:35:31,708 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:31,708 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:31,709 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124 after 1ms
2024-11-19T07:35:31,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741838_1019 (size=1264)
2024-11-19T07:35:31,709 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:31,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741838_1019 (size=1264)
2024-11-19T07:35:31,710 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 is not closed yet, will try archiving it next time
2024-11-19T07:35:31,718 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732001711205/Put/vlen=218/seqid=0]
2024-11-19T07:35:31,718 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732001720956/Put/vlen=1045/seqid=0]
2024-11-19T07:35:31,719 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001710124
2024-11-19T07:35:31,719 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:31,719 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:31,719 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 after 0ms
2024-11-19T07:35:31,719 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:31,723 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732001724327/Put/vlen=1045/seqid=0]
2024-11-19T07:35:31,723 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732001726343/Put/vlen=1045/seqid=0]
2024-11-19T07:35:31,723 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328
2024-11-19T07:35:31,723 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:31,723 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:31,724 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690 after 1ms
2024-11-19T07:35:31,724 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001729690
2024-11-19T07:35:31,727 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732001729689/Put/vlen=1045/seqid=0]
2024-11-19T07:35:31,727 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:31,727 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:31,728 WARN  [IPC Server handler 2 on default port 34215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-19T07:35:31,728 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 after 1ms
2024-11-19T07:35:32,093 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:32,193 WARN  [ResponseProcessor for block BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:32,193 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1323575647_22 at /127.0.0.1:39752 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36933:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39752 dst: /127.0.0.1:36933
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36933 remote=/127.0.0.1:39752]. Total timeout mills is 60000, 59513 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:32,193 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 block BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36933,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK], DatanodeInfoWithStorage[127.0.0.1:43771,DS-fd076b65-1660-424c-a802-4a7f1eb23f56,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36933,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]) is bad.
2024-11-19T07:35:32,193 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1323575647_22 at /127.0.0.1:57136 [Receiving block BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:43771:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57136 dst: /127.0.0.1:43771
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:35:32,195 WARN  [DataStreamer for file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 block BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:32,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741839_1022 (size=85)
2024-11-19T07:35:32,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:33,094 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:33,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:33,701 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001724328 after 4002ms
2024-11-19T07:35:34,095 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:34,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:35,030 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T07:35:35,095 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:35,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:35,729 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 after 4002ms
2024-11-19T07:35:35,729 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:35,735 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:35,735 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 94dbd9302ccd7473466748db5c92e1c9 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-19T07:35:35,735 ERROR [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,736 WARN  [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,737 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C38857%2C1732001709382:(num 1732001731701) roll requested
2024-11-19T07:35:35,737 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.1732001735737
2024-11-19T07:35:35,743 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 newFile=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001735737
2024-11-19T07:35:35,743 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,743 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,744 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,744 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,744 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,744 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001735737
2024-11-19T07:35:35,744 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,744 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1016362256-172.17.0.2-1732001707159:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,745 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:35,745 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 after 0ms
2024-11-19T07:35:35,745 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34413:34413),(127.0.0.1/127.0.0.1:36257:36257)]
2024-11-19T07:35:35,746 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.1732001731701 to hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs/db0afee3eab9%2C38857%2C1732001709382.1732001731701
2024-11-19T07:35:35,767 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/.tmp/info/19e592c619f94b16aab9b6f135b5c57f is 1080, key is row1002/info:/1732001720956/Put/seqid=0
2024-11-19T07:35:35,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741841_1024 (size=9270)
2024-11-19T07:35:35,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741841_1024 (size=9270)
2024-11-19T07:35:35,773 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/.tmp/info/19e592c619f94b16aab9b6f135b5c57f
2024-11-19T07:35:35,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/.tmp/info/19e592c619f94b16aab9b6f135b5c57f as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/info/19e592c619f94b16aab9b6f135b5c57f
2024-11-19T07:35:35,786 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/info/19e592c619f94b16aab9b6f135b5c57f, entries=4, sequenceid=8, filesize=9.1 K
2024-11-19T07:35:35,787 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 94dbd9302ccd7473466748db5c92e1c9 in 52ms, sequenceid=8, compaction requested=false
2024-11-19T07:35:35,787 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 94dbd9302ccd7473466748db5c92e1c9:

2024-11-19T07:35:35,787 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-19T07:35:35,788 ERROR [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,788 WARN  [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b-prefix:db0afee3eab9,38857,1732001709382.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,788 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C38857%2C1732001709382.meta:.meta(num 1732001710685) roll requested
2024-11-19T07:35:35,788 INFO  [regionserver/db0afee3eab9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C38857%2C1732001709382.meta.1732001735788.meta
2024-11-19T07:35:35,796 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,796 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,796 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,796 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,796 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:35,797 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001735788.meta
2024-11-19T07:35:35,797 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,797 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:35,797 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta
2024-11-19T07:35:35,798 WARN  [IPC Server handler 4 on default port 34215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-11-19T07:35:35,798 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36257:36257),(127.0.0.1/127.0.0.1:34413:34413)]
2024-11-19T07:35:35,798 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta after 1ms
2024-11-19T07:35:35,798 DEBUG [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta is not closed yet, will try archiving it next time
2024-11-19T07:35:35,820 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/info/f739b3445c0e4e15a9eaa87c2f0c3d7a is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9./info:regioninfo/1732001711210/Put/seqid=0
2024-11-19T07:35:35,825 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741843_1027 (size=7125)
2024-11-19T07:35:35,825 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741843_1027 (size=7125)
2024-11-19T07:35:35,826 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/info/f739b3445c0e4e15a9eaa87c2f0c3d7a
2024-11-19T07:35:35,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/ns/80d9e52e3c214b91990c3feeb0eb2e1c is 43, key is default/ns:d/1732001710762/Put/seqid=0
2024-11-19T07:35:35,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741844_1028 (size=5153)
2024-11-19T07:35:35,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741844_1028 (size=5153)
2024-11-19T07:35:35,852 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/ns/80d9e52e3c214b91990c3feeb0eb2e1c
2024-11-19T07:35:35,874 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/table/8392c55dc90042298a8e4d7cee89a2a2 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732001711221/Put/seqid=0
2024-11-19T07:35:35,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741845_1029 (size=5438)
2024-11-19T07:35:35,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741845_1029 (size=5438)
2024-11-19T07:35:35,879 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/table/8392c55dc90042298a8e4d7cee89a2a2
2024-11-19T07:35:35,885 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/info/f739b3445c0e4e15a9eaa87c2f0c3d7a as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/info/f739b3445c0e4e15a9eaa87c2f0c3d7a
2024-11-19T07:35:35,892 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/info/f739b3445c0e4e15a9eaa87c2f0c3d7a, entries=10, sequenceid=11, filesize=7.0 K
2024-11-19T07:35:35,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/ns/80d9e52e3c214b91990c3feeb0eb2e1c as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/ns/80d9e52e3c214b91990c3feeb0eb2e1c
2024-11-19T07:35:35,899 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/ns/80d9e52e3c214b91990c3feeb0eb2e1c, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T07:35:35,900 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/.tmp/table/8392c55dc90042298a8e4d7cee89a2a2 as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/table/8392c55dc90042298a8e4d7cee89a2a2
2024-11-19T07:35:35,907 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/table/8392c55dc90042298a8e4d7cee89a2a2, entries=2, sequenceid=11, filesize=5.3 K
2024-11-19T07:35:35,908 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 121ms, sequenceid=11, compaction requested=false
2024-11-19T07:35:35,908 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T07:35:35,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:35:35,914 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:35:35,915 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:35:35,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:35,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:35,915 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:35:35,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:35:35,915 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1394392011, stopped=false
2024-11-19T07:35:35,915 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,41197,1732001709217
2024-11-19T07:35:35,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:35,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:35,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:35,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:35,984 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:35:35,985 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:35:35,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:35:35,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:35,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:35,985 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,38857,1732001709382' *****
2024-11-19T07:35:35,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:35,986 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:35:35,986 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:35:35,986 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:35:35,986 INFO  [RS:0;db0afee3eab9:38857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:35:35,986 INFO  [RS:0;db0afee3eab9:38857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:35:35,986 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(3091): Received CLOSE for 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,38857,1732001709382
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:38857.
2024-11-19T07:35:35,987 DEBUG [RS:0;db0afee3eab9:38857 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:35:35,987 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94dbd9302ccd7473466748db5c92e1c9, disabling compactions & flushes
2024-11-19T07:35:35,987 DEBUG [RS:0;db0afee3eab9:38857 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:35,987 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:35,987 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:35,987 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9. after waiting 0 ms
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:35:35,987 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:35:35,987 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:35:35,988 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T07:35:35,988 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1325): Online Regions={94dbd9302ccd7473466748db5c92e1c9=TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9., 1588230740=hbase:meta,,1.1588230740}
2024-11-19T07:35:35,988 DEBUG [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 94dbd9302ccd7473466748db5c92e1c9
2024-11-19T07:35:35,988 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:35:35,989 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:35:35,989 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:35:35,989 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:35:35,989 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:35:35,990 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:35:35,994 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/default/TestLogRolling-testLogRollOnPipelineRestart/94dbd9302ccd7473466748db5c92e1c9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-19T07:35:35,994 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:35,995 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94dbd9302ccd7473466748db5c92e1c9:
Waiting for close lock at 1732001735987Running coprocessor pre-close hooks at 1732001735987Disabling compacts and flushes for region at 1732001735987Disabling writes for close at 1732001735987Writing region close event to WAL at 1732001735989 (+2 ms)Running coprocessor post-close hooks at 1732001735994 (+5 ms)Closed at 1732001735994
2024-11-19T07:35:35,995 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732001710845.94dbd9302ccd7473466748db5c92e1c9.
2024-11-19T07:35:35,995 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T07:35:35,996 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:35:35,996 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:35:35,996 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001735988Running coprocessor pre-close hooks at 1732001735988Disabling compacts and flushes for region at 1732001735988Disabling writes for close at 1732001735989 (+1 ms)Writing region close event to WAL at 1732001735990 (+1 ms)Running coprocessor post-close hooks at 1732001735996 (+6 ms)Closed at 1732001735996
2024-11-19T07:35:35,996 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:35:36,011 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T07:35:36,011 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T07:35:36,096 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:36,188 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,38857,1732001709382; all regions closed.
2024-11-19T07:35:36,188 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:36,189 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:36,189 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:36,189 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:36,189 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:36,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741842_1025 (size=825)
2024-11-19T07:35:36,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741842_1025 (size=825)
2024-11-19T07:35:36,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:37,097 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:37,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:38,098 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:38,189 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T07:35:38,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:39,098 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:39,199 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:35:39,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:39,799 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta after 4002ms
2024-11-19T07:35:39,799 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/WALs/db0afee3eab9,38857,1732001709382/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta to hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs/db0afee3eab9%2C38857%2C1732001709382.meta.1732001710685.meta
2024-11-19T07:35:39,802 DEBUG [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs
2024-11-19T07:35:39,802 INFO  [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C38857%2C1732001709382.meta:.meta(num 1732001735788)
2024-11-19T07:35:39,802 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,802 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,802 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,802 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,802 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741840_1023 (size=1162)
2024-11-19T07:35:39,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741840_1023 (size=1162)
2024-11-19T07:35:39,809 DEBUG [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs
2024-11-19T07:35:39,809 INFO  [RS:0;db0afee3eab9:38857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C38857%2C1732001709382:(num 1732001735737)
2024-11-19T07:35:39,809 DEBUG [RS:0;db0afee3eab9:38857 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:39,809 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:35:39,809 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:35:39,809 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:35:39,810 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:35:39,810 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:35:39,810 INFO  [RS:0;db0afee3eab9:38857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38857
2024-11-19T07:35:39,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,38857,1732001709382
2024-11-19T07:35:39,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:35:39,857 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:35:39,868 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,38857,1732001709382]
2024-11-19T07:35:39,878 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,38857,1732001709382 already deleted, retry=false
2024-11-19T07:35:39,879 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,38857,1732001709382 expired; onlineServers=0
2024-11-19T07:35:39,879 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,41197,1732001709217' *****
2024-11-19T07:35:39,879 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:35:39,879 INFO  [M:0;db0afee3eab9:41197 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:35:39,879 INFO  [M:0;db0afee3eab9:41197 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:35:39,879 DEBUG [M:0;db0afee3eab9:41197 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:35:39,879 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:35:39,879 DEBUG [M:0;db0afee3eab9:41197 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:35:39,879 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001709868 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001709868,5,FailOnTimeoutGroup]
2024-11-19T07:35:39,879 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001709868 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001709868,5,FailOnTimeoutGroup]
2024-11-19T07:35:39,880 INFO  [M:0;db0afee3eab9:41197 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:35:39,880 INFO  [M:0;db0afee3eab9:41197 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:35:39,880 DEBUG [M:0;db0afee3eab9:41197 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:35:39,880 INFO  [M:0;db0afee3eab9:41197 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:35:39,880 INFO  [M:0;db0afee3eab9:41197 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:35:39,880 INFO  [M:0;db0afee3eab9:41197 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:35:39,880 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:35:39,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:35:39,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:39,889 DEBUG [M:0;db0afee3eab9:41197 {}] zookeeper.ZKUtil(347): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:35:39,889 WARN  [M:0;db0afee3eab9:41197 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:35:39,890 INFO  [M:0;db0afee3eab9:41197 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/.lastflushedseqids
2024-11-19T07:35:39,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741846_1030 (size=111)
2024-11-19T07:35:39,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741846_1030 (size=111)
2024-11-19T07:35:39,897 INFO  [M:0;db0afee3eab9:41197 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:35:39,898 INFO  [M:0;db0afee3eab9:41197 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:35:39,898 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:35:39,898 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:39,898 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:39,898 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:35:39,898 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:39,898 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-19T07:35:39,898 ERROR [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData-prefix:db0afee3eab9,41197,1732001709217 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:39,899 WARN  [FSHLog-0-hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData-prefix:db0afee3eab9,41197,1732001709217 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:39,899 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog db0afee3eab9%2C41197%2C1732001709217:(num 1732001709529) roll requested
2024-11-19T07:35:39,899 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C41197%2C1732001709217.1732001739899
2024-11-19T07:35:39,904 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,904 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,904 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,905 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,905 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:39,905 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001739899
2024-11-19T07:35:39,905 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:39,905 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35141,DS-c5074078-4407-43da-8052-4cc1948c719f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-19T07:35:39,905 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529
2024-11-19T07:35:39,906 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36257:36257),(127.0.0.1/127.0.0.1:34413:34413)]
2024-11-19T07:35:39,906 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 is not closed yet, will try archiving it next time
2024-11-19T07:35:39,906 WARN  [IPC Server handler 4 on default port 34215 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-19T07:35:39,906 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 after 1ms
2024-11-19T07:35:39,926 DEBUG [M:0;db0afee3eab9:41197 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1babc3869b140bd817f95ae09ac50e8 is 82, key is hbase:meta,,1/info:regioninfo/1732001710713/Put/seqid=0
2024-11-19T07:35:39,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741848_1033 (size=5672)
2024-11-19T07:35:39,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741848_1033 (size=5672)
2024-11-19T07:35:39,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:39,968 INFO  [RS:0;db0afee3eab9:38857 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:35:39,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1015204b83b0001, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:39,968 INFO  [RS:0;db0afee3eab9:38857 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,38857,1732001709382; zookeeper connection closed.
2024-11-19T07:35:39,968 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@448d0634 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@448d0634
2024-11-19T07:35:39,969 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:35:40,099 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:40,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:40,335 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1babc3869b140bd817f95ae09ac50e8
2024-11-19T07:35:40,363 DEBUG [M:0;db0afee3eab9:41197 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd584193028e444185e098b8f44d1683 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732001711226/Put/seqid=0
2024-11-19T07:35:40,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741849_1034 (size=6118)
2024-11-19T07:35:40,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741849_1034 (size=6118)
2024-11-19T07:35:40,368 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd584193028e444185e098b8f44d1683
2024-11-19T07:35:40,390 DEBUG [M:0;db0afee3eab9:41197 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/15c45c6bbec842f9bff6037b20de18f0 is 69, key is db0afee3eab9,38857,1732001709382/rs:state/1732001709963/Put/seqid=0
2024-11-19T07:35:40,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741850_1035 (size=5156)
2024-11-19T07:35:40,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741850_1035 (size=5156)
2024-11-19T07:35:40,399 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/15c45c6bbec842f9bff6037b20de18f0
2024-11-19T07:35:40,418 DEBUG [M:0;db0afee3eab9:41197 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11ead72faaea418e9d4759a1522d118b is 52, key is load_balancer_on/state:d/1732001710838/Put/seqid=0
2024-11-19T07:35:40,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741851_1036 (size=5056)
2024-11-19T07:35:40,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741851_1036 (size=5056)
2024-11-19T07:35:40,423 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11ead72faaea418e9d4759a1522d118b
2024-11-19T07:35:40,429 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a1babc3869b140bd817f95ae09ac50e8 as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a1babc3869b140bd817f95ae09ac50e8
2024-11-19T07:35:40,434 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a1babc3869b140bd817f95ae09ac50e8, entries=8, sequenceid=56, filesize=5.5 K
2024-11-19T07:35:40,436 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd584193028e444185e098b8f44d1683 as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd584193028e444185e098b8f44d1683
2024-11-19T07:35:40,442 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd584193028e444185e098b8f44d1683, entries=6, sequenceid=56, filesize=6.0 K
2024-11-19T07:35:40,443 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/15c45c6bbec842f9bff6037b20de18f0 as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/15c45c6bbec842f9bff6037b20de18f0
2024-11-19T07:35:40,448 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/15c45c6bbec842f9bff6037b20de18f0, entries=1, sequenceid=56, filesize=5.0 K
2024-11-19T07:35:40,449 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/11ead72faaea418e9d4759a1522d118b as hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/11ead72faaea418e9d4759a1522d118b
2024-11-19T07:35:40,454 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/11ead72faaea418e9d4759a1522d118b, entries=1, sequenceid=56, filesize=4.9 K
2024-11-19T07:35:40,455 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 557ms, sequenceid=56, compaction requested=false
2024-11-19T07:35:40,456 INFO  [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:40,456 DEBUG [M:0;db0afee3eab9:41197 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001739898Disabling compacts and flushes for region at 1732001739898Disabling writes for close at 1732001739898Obtaining lock to block concurrent updates at 1732001739898Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001739898Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732001739899 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001739906 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001739906Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001739925 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001739925Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001740341 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001740363 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001740363Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001740374 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001740389 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001740389Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001740403 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001740417 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001740417Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f15e3f: reopening flushed file at 1732001740428 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66c6ec4d: reopening flushed file at 1732001740435 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@216285a7: reopening flushed file at 1732001740442 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e4b95b8: reopening flushed file at 1732001740448 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 557ms, sequenceid=56, compaction requested=false at 1732001740455 (+7 ms)Writing region close event to WAL at 1732001740456 (+1 ms)Closed at 1732001740456
2024-11-19T07:35:40,457 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:40,457 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:40,457 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:40,457 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:40,457 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:35:40,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741847_1031 (size=757)
2024-11-19T07:35:40,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43771 is added to blk_1073741847_1031 (size=757)
2024-11-19T07:35:40,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:40,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,018 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,024 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,025 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,100 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:41,189 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-19T07:35:41,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:41,527 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:41,528 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,528 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,529 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,529 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,546 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,552 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,554 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:41,559 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:35:41,559 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:35:41,559 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:35:41,560 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-19T07:35:42,100 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:42,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:43,101 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:43,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:43,907 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 after 4002ms
2024-11-19T07:35:43,907 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/WALs/db0afee3eab9,41197,1732001709217/db0afee3eab9%2C41197%2C1732001709217.1732001709529 to hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/oldWALs/db0afee3eab9%2C41197%2C1732001709217.1732001709529
2024-11-19T07:35:43,912 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/MasterData/oldWALs/db0afee3eab9%2C41197%2C1732001709217.1732001709529 to hdfs://localhost:34215/user/jenkins/test-data/f9863168-ed8b-265d-70ff-6a609aba8e9b/oldWALs/db0afee3eab9%2C41197%2C1732001709217.1732001709529$masterlocalwal$
2024-11-19T07:35:43,912 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:35:43,912 INFO  [M:0;db0afee3eab9:41197 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:35:43,912 INFO  [M:0;db0afee3eab9:41197 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41197
2024-11-19T07:35:43,913 INFO  [M:0;db0afee3eab9:41197 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:35:44,063 INFO  [M:0;db0afee3eab9:41197 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:35:44,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:44,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41197-0x1015204b83b0000, quorum=127.0.0.1:53294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:35:44,066 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c524311{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:44,066 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@523d3401{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:44,066 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:44,066 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3420abff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:44,066 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f3e5a16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:44,068 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:44,068 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:44,068 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid c285a081-1271-4a07-b660-ba1b01062ae8) service to localhost/127.0.0.1:34215
2024-11-19T07:35:44,068 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:44,069 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data3/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:44,069 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data4/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:44,069 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:44,071 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a8b2822{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:44,072 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68df0564{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:44,072 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:44,072 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6573e60c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:44,072 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ef50a45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:44,074 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:35:44,074 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:35:44,074 WARN  [BP-1016362256-172.17.0.2-1732001707159 heartbeating to localhost/127.0.0.1:34215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1016362256-172.17.0.2-1732001707159 (Datanode Uuid 50168649-4975-48fe-86c3-cb95fec33746) service to localhost/127.0.0.1:34215
2024-11-19T07:35:44,074 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:35:44,075 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data1/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:44,075 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/cluster_ba0f4546-cdad-947e-4d23-633dd58eb7da/data/data2/current/BP-1016362256-172.17.0.2-1732001707159 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:35:44,076 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:35:44,081 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@232fa1ae{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:35:44,082 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c07fc0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:35:44,082 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:35:44,082 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10c583a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:35:44,082 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7096145a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir/,STOPPED}
2024-11-19T07:35:44,088 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:35:44,101 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:44,108 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:35:44,118 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=181 (was 154)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34215 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34215 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34215
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:34215
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34215 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 429) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=11344 (was 11582)
2024-11-19T07:35:44,126 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=181, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=11343
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.log.dir so I do NOT create it in target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1eb3a90c-98be-b4e2-cfab-966902afec32/hadoop.tmp.dir so I do NOT create it in target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119, deleteOnExit=true
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/test.cache.data in system properties and HBase conf
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:35:44,127 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:35:44,128 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:35:44,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:35:44,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:35:44,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:35:44,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:35:44,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:35:44,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:35:44,143 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:35:44,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:44,595 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:44,599 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:44,600 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:44,600 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:44,600 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:44,601 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:44,601 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77877788{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:44,601 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c020752{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:44,707 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fc981fd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/java.io.tmpdir/jetty-localhost-45269-hadoop-hdfs-3_4_1-tests_jar-_-any-9764384929915398573/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:35:44,708 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4732430a{HTTP/1.1, (http/1.1)}{localhost:45269}
2024-11-19T07:35:44,708 INFO  [Time-limited test {}] server.Server(415): Started @201396ms
2024-11-19T07:35:44,721 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:35:44,947 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:44,951 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:44,951 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:44,952 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:44,952 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:44,957 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6c1f86{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:44,957 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@261a9e0a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:45,066 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37d74326{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/java.io.tmpdir/jetty-localhost-39507-hadoop-hdfs-3_4_1-tests_jar-_-any-1414183099576534704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:45,067 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@77f859f0{HTTP/1.1, (http/1.1)}{localhost:39507}
2024-11-19T07:35:45,067 INFO  [Time-limited test {}] server.Server(415): Started @201756ms
2024-11-19T07:35:45,068 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:45,097 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:35:45,101 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:35:45,102 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:35:45,102 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:35:45,102 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:35:45,102 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38d3f6f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:35:45,102 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:45,102 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36f52998{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:35:45,207 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fc2e521{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/java.io.tmpdir/jetty-localhost-40921-hadoop-hdfs-3_4_1-tests_jar-_-any-3560529783230290302/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:35:45,207 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bff2115{HTTP/1.1, (http/1.1)}{localhost:40921}
2024-11-19T07:35:45,208 INFO  [Time-limited test {}] server.Server(415): Started @201896ms
2024-11-19T07:35:45,209 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:35:45,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:46,102 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:46,164 WARN  [Thread-1672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data1/current/BP-1081112191-172.17.0.2-1732001744154/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:46,164 WARN  [Thread-1673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data2/current/BP-1081112191-172.17.0.2-1732001744154/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:46,198 WARN  [Thread-1636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:46,200 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a450a3a99b0473 with lease ID 0x292770038dc3ac02: Processing first storage report for DS-d53f91bb-9bee-44f9-b2dd-d0c40e5ddabb from datanode DatanodeRegistration(127.0.0.1:45523, datanodeUuid=58c7e4f9-f099-45ac-ac72-cf614bb91207, infoPort=39355, infoSecurePort=0, ipcPort=38437, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154)
2024-11-19T07:35:46,200 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a450a3a99b0473 with lease ID 0x292770038dc3ac02: from storage DS-d53f91bb-9bee-44f9-b2dd-d0c40e5ddabb node DatanodeRegistration(127.0.0.1:45523, datanodeUuid=58c7e4f9-f099-45ac-ac72-cf614bb91207, infoPort=39355, infoSecurePort=0, ipcPort=38437, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:46,201 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a450a3a99b0473 with lease ID 0x292770038dc3ac02: Processing first storage report for DS-f96961bd-87b2-42ed-9be1-c5d88af3a92a from datanode DatanodeRegistration(127.0.0.1:45523, datanodeUuid=58c7e4f9-f099-45ac-ac72-cf614bb91207, infoPort=39355, infoSecurePort=0, ipcPort=38437, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154)
2024-11-19T07:35:46,201 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a450a3a99b0473 with lease ID 0x292770038dc3ac02: from storage DS-f96961bd-87b2-42ed-9be1-c5d88af3a92a node DatanodeRegistration(127.0.0.1:45523, datanodeUuid=58c7e4f9-f099-45ac-ac72-cf614bb91207, infoPort=39355, infoSecurePort=0, ipcPort=38437, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:46,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:46,359 WARN  [Thread-1684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data4/current/BP-1081112191-172.17.0.2-1732001744154/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:46,359 WARN  [Thread-1683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data3/current/BP-1081112191-172.17.0.2-1732001744154/current, will proceed with Du for space computation calculation, 
2024-11-19T07:35:46,384 WARN  [Thread-1659 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:35:46,386 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b92629192102fbd with lease ID 0x292770038dc3ac03: Processing first storage report for DS-6f4861ad-b978-48b5-b13e-8da0de85ae70 from datanode DatanodeRegistration(127.0.0.1:36715, datanodeUuid=ce418414-4e71-4f93-ab4a-eb5bd8f4ca4d, infoPort=39149, infoSecurePort=0, ipcPort=39291, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154)
2024-11-19T07:35:46,387 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b92629192102fbd with lease ID 0x292770038dc3ac03: from storage DS-6f4861ad-b978-48b5-b13e-8da0de85ae70 node DatanodeRegistration(127.0.0.1:36715, datanodeUuid=ce418414-4e71-4f93-ab4a-eb5bd8f4ca4d, infoPort=39149, infoSecurePort=0, ipcPort=39291, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:46,387 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b92629192102fbd with lease ID 0x292770038dc3ac03: Processing first storage report for DS-f53c845c-1667-4bef-bb35-8cbe983db487 from datanode DatanodeRegistration(127.0.0.1:36715, datanodeUuid=ce418414-4e71-4f93-ab4a-eb5bd8f4ca4d, infoPort=39149, infoSecurePort=0, ipcPort=39291, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154)
2024-11-19T07:35:46,387 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b92629192102fbd with lease ID 0x292770038dc3ac03: from storage DS-f53c845c-1667-4bef-bb35-8cbe983db487 node DatanodeRegistration(127.0.0.1:36715, datanodeUuid=ce418414-4e71-4f93-ab4a-eb5bd8f4ca4d, infoPort=39149, infoSecurePort=0, ipcPort=39291, storageInfo=lv=-57;cid=testClusterID;nsid=1705575742;c=1732001744154), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:35:46,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260
2024-11-19T07:35:46,444 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/zookeeper_0, clientPort=62849, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:35:46,445 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62849
2024-11-19T07:35:46,445 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,446 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:35:46,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:35:46,456 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a with version=8
2024-11-19T07:35:46,456 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:35:46,459 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:35:46,460 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:35:46,460 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33873
2024-11-19T07:35:46,462 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33873 connecting to ZooKeeper ensemble=127.0.0.1:62849
2024-11-19T07:35:46,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:338730x0, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:35:46,518 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33873-0x101520549b60000 connected
2024-11-19T07:35:46,697 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,699 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:46,702 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a, hbase.cluster.distributed=false
2024-11-19T07:35:46,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:35:46,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33873
2024-11-19T07:35:46,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33873
2024-11-19T07:35:46,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33873
2024-11-19T07:35:46,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33873
2024-11-19T07:35:46,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33873
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:35:46,729 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:35:46,730 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:35:46,730 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37281
2024-11-19T07:35:46,732 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37281 connecting to ZooKeeper ensemble=127.0.0.1:62849
2024-11-19T07:35:46,733 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,735 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372810x0, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:35:46,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372810x0, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:35:46,763 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:35:46,764 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37281-0x101520549b60001 connected
2024-11-19T07:35:46,764 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:35:46,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:35:46,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:35:46,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37281
2024-11-19T07:35:46,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37281
2024-11-19T07:35:46,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37281
2024-11-19T07:35:46,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37281
2024-11-19T07:35:46,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37281
2024-11-19T07:35:46,784 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:33873
2024-11-19T07:35:46,784 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,33873,1732001746459
2024-11-19T07:35:46,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:46,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:46,795 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,33873,1732001746459
2024-11-19T07:35:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:35:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:46,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:46,806 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:35:46,807 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,33873,1732001746459 from backup master directory
2024-11-19T07:35:46,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:46,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,33873,1732001746459
2024-11-19T07:35:46,815 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:35:46,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:35:46,815 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,33873,1732001746459
2024-11-19T07:35:46,828 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/hbase.id] with ID: ce11835e-c41d-451a-85d8-a50570f55d5f
2024-11-19T07:35:46,828 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/.tmp/hbase.id
2024-11-19T07:35:46,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:35:46,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:35:46,843 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/.tmp/hbase.id]:[hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/hbase.id]
2024-11-19T07:35:46,867 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:46,868 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:35:46,869 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-19T07:35:46,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:46,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:46,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:35:46,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:35:46,895 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:35:46,896 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:35:46,897 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:46,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:35:46,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:35:46,911 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store
2024-11-19T07:35:46,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:35:46,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:35:46,922 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:46,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:35:46,923 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:46,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:46,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:35:46,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:46,923 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:35:46,923 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001746923Disabling compacts and flushes for region at 1732001746923Disabling writes for close at 1732001746923Writing region close event to WAL at 1732001746923Closed at 1732001746923
2024-11-19T07:35:46,924 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/.initializing
2024-11-19T07:35:46,924 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/WALs/db0afee3eab9,33873,1732001746459
2024-11-19T07:35:46,926 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C33873%2C1732001746459, suffix=, logDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/WALs/db0afee3eab9,33873,1732001746459, archiveDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/oldWALs, maxLogs=10
2024-11-19T07:35:46,927 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C33873%2C1732001746459.1732001746927
2024-11-19T07:35:46,933 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/WALs/db0afee3eab9,33873,1732001746459/db0afee3eab9%2C33873%2C1732001746459.1732001746927
2024-11-19T07:35:46,937 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39149:39149),(127.0.0.1/127.0.0.1:39355:39355)]
2024-11-19T07:35:46,940 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:46,941 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:46,941 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,941 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,944 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:46,946 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:35:46,946 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:46,947 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:46,947 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:46,948 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:35:46,948 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:46,949 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:46,949 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:46,950 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:35:46,950 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:46,951 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:46,951 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:35:46,952 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:35:46,953 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:46,953 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:46,953 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,954 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,955 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,956 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,957 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,958 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:35:46,959 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:35:46,963 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:46,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=828649, jitterRate=0.05368274450302124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:35:46,964 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001746941Initializing all the Stores at 1732001746942 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001746942Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001746944 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001746944Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001746944Cleaning up temporary data from old regions at 1732001746957 (+13 ms)Region opened successfully at 1732001746964 (+7 ms)
2024-11-19T07:35:46,968 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:35:46,972 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30be07e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:35:46,974 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:35:46,974 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:35:46,974 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:35:46,974 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:35:46,975 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:35:46,976 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:35:46,976 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:35:46,982 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:35:46,984 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:35:46,994 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:35:46,995 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:35:46,998 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:35:47,004 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:35:47,005 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:35:47,007 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:35:47,015 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:35:47,016 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:35:47,025 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:35:47,029 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:35:47,036 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:35:47,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:47,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,048 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,33873,1732001746459, sessionid=0x101520549b60000, setting cluster-up flag (Was=false)
2024-11-19T07:35:47,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:35:47,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,062 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:47,063 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,063 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,064 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,064 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,064 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,064 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,089 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,090 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,090 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,090 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,097 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,098 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,098 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,099 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:35:47,101 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,33873,1732001746459
2024-11-19T07:35:47,102 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:47,103 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:47,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,152 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:35:47,153 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,33873,1732001746459
2024-11-19T07:35:47,155 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:35:47,158 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:47,159 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:35:47,159 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:35:47,161 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,33873,1732001746459
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:35:47,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,174 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(746): ClusterId : ce11835e-c41d-451a-85d8-a50570f55d5f
2024-11-19T07:35:47,174 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:35:47,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:47,179 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:35:47,181 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:47,181 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:35:47,186 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:35:47,187 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:35:47,188 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001777188
2024-11-19T07:35:47,188 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:35:47,189 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:35:47,189 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:35:47,189 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:35:47,189 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:35:47,189 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:35:47,195 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:35:47,195 DEBUG [RS:0;db0afee3eab9:37281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f4e2c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:35:47,196 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,209 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:35:47,209 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:35:47,209 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:35:47,211 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:35:47,211 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:35:47,211 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001747211,5,FailOnTimeoutGroup]
2024-11-19T07:35:47,211 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001747211,5,FailOnTimeoutGroup]
2024-11-19T07:35:47,212 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,212 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:35:47,212 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,212 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:35:47,226 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:37281
2024-11-19T07:35:47,226 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:35:47,226 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:35:47,226 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:35:47,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:35:47,227 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,33873,1732001746459 with port=37281, startcode=1732001746728
2024-11-19T07:35:47,228 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:35:47,228 DEBUG [RS:0;db0afee3eab9:37281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:35:47,228 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a
2024-11-19T07:35:47,234 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:35:47,234 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33873 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,234 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33873 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,238 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a
2024-11-19T07:35:47,238 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44151
2024-11-19T07:35:47,238 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:35:47,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:35:47,247 DEBUG [RS:0;db0afee3eab9:37281 {}] zookeeper.ZKUtil(111): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,248 WARN  [RS:0;db0afee3eab9:37281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:35:47,248 INFO  [RS:0;db0afee3eab9:37281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:47,248 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,249 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,37281,1732001746728]
2024-11-19T07:35:47,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:35:47,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:35:47,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:47,254 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:35:47,256 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:35:47,256 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:47,257 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:47,257 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:35:47,258 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:35:47,259 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:35:47,259 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:35:47,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:47,260 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:47,260 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:35:47,260 INFO  [RS:0;db0afee3eab9:37281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:35:47,260 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,262 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:35:47,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:47,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:47,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:35:47,264 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:35:47,265 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:35:47,265 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:35:47,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:47,265 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,265 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,265 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,265 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,265 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:35:47,266 DEBUG [RS:0;db0afee3eab9:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:35:47,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740
2024-11-19T07:35:47,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740
2024-11-19T07:35:47,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:35:47,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:35:47,270 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:35:47,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:35:47,276 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,276 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,276 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,276 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,276 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,277 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37281,1732001746728-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:35:47,281 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:47,282 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706449, jitterRate=-0.10170483589172363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:35:47,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001747252Initializing all the Stores at 1732001747253 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001747253Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001747254 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001747254Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001747254Cleaning up temporary data from old regions at 1732001747270 (+16 ms)Region opened successfully at 1732001747282 (+12 ms)
2024-11-19T07:35:47,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:35:47,282 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:35:47,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:35:47,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:35:47,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:35:47,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:47,288 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:35:47,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001747282Disabling compacts and flushes for region at 1732001747282Disabling writes for close at 1732001747283 (+1 ms)Writing region close event to WAL at 1732001747288 (+5 ms)Closed at 1732001747288
2024-11-19T07:35:47,290 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:47,290 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:35:47,290 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:35:47,293 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:35:47,299 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:35:47,312 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:35:47,312 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37281,1732001746728-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,312 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,312 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.Replication(171): db0afee3eab9,37281,1732001746728 started
2024-11-19T07:35:47,334 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:47,334 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,37281,1732001746728, RpcServer on db0afee3eab9/172.17.0.2:37281, sessionid=0x101520549b60001
2024-11-19T07:35:47,335 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:35:47,335 DEBUG [RS:0;db0afee3eab9:37281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,335 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37281,1732001746728'
2024-11-19T07:35:47,335 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:35:47,336 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,37281,1732001746728'
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:35:47,337 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:35:47,338 DEBUG [RS:0;db0afee3eab9:37281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:35:47,338 INFO  [RS:0;db0afee3eab9:37281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:35:47,338 INFO  [RS:0;db0afee3eab9:37281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:35:47,440 INFO  [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37281%2C1732001746728, suffix=, logDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728, archiveDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs, maxLogs=32
2024-11-19T07:35:47,440 INFO  [RS:0;db0afee3eab9:37281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37281%2C1732001746728.1732001747440
2024-11-19T07:35:47,450 WARN  [db0afee3eab9:33873 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:35:47,453 INFO  [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001747440
2024-11-19T07:35:47,463 DEBUG [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39355:39355),(127.0.0.1/127.0.0.1:39149:39149)]
2024-11-19T07:35:47,700 DEBUG [db0afee3eab9:33873 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:35:47,701 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:47,702 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,37281,1732001746728, state=OPENING
2024-11-19T07:35:47,838 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:35:47,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:35:47,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:47,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:47,900 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:35:47,900 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,37281,1732001746728}]
2024-11-19T07:35:48,054 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:35:48,056 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59005, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:35:48,061 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:35:48,061 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:35:48,063 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37281%2C1732001746728.meta, suffix=.meta, logDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728, archiveDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs, maxLogs=32
2024-11-19T07:35:48,064 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37281%2C1732001746728.meta.1732001748063.meta
2024-11-19T07:35:48,069 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.meta.1732001748063.meta
2024-11-19T07:35:48,070 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39355:39355),(127.0.0.1/127.0.0.1:39149:39149)]
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:35:48,071 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:35:48,071 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:35:48,073 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:35:48,074 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:35:48,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:48,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:35:48,076 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:35:48,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,076 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:48,076 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:35:48,077 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:35:48,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:48,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:35:48,079 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:35:48,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,080 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:35:48,080 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:35:48,081 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740
2024-11-19T07:35:48,082 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740
2024-11-19T07:35:48,083 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:35:48,084 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:35:48,084 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:35:48,086 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:35:48,087 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=853558, jitterRate=0.08535610139369965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:35:48,088 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:35:48,088 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001748072Writing region info on filesystem at 1732001748072Initializing all the Stores at 1732001748072Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001748073 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001748073Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001748073Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001748073Cleaning up temporary data from old regions at 1732001748084 (+11 ms)Running coprocessor post-open hooks at 1732001748088 (+4 ms)Region opened successfully at 1732001748088
2024-11-19T07:35:48,090 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001748053
2024-11-19T07:35:48,093 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:35:48,093 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:35:48,096 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:48,097 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,37281,1732001746728, state=OPEN
2024-11-19T07:35:48,104 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:48,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:35:48,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:35:48,140 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:48,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:48,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:35:48,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:35:48,145 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,37281,1732001746728 in 240 msec
2024-11-19T07:35:48,150 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:35:48,150 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 855 msec
2024-11-19T07:35:48,152 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:35:48,152 INFO  [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:35:48,154 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:35:48,154 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37281,1732001746728, seqNum=-1]
2024-11-19T07:35:48,155 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:35:48,156 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49225, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:35:48,164 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0050 sec
2024-11-19T07:35:48,164 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001748164, completionTime=-1
2024-11-19T07:35:48,164 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:35:48,164 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:35:48,166 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:35:48,166 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001808166
2024-11-19T07:35:48,166 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001868166
2024-11-19T07:35:48,166 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:33873, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,167 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,171 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:35:48,173 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.358sec
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:35:48,174 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:35:48,177 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:35:48,177 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:35:48,177 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,33873,1732001746459-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:35:48,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acc1c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:48,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,33873,-1 for getting cluster id
2024-11-19T07:35:48,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:35:48,279 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce11835e-c41d-451a-85d8-a50570f55d5f'
2024-11-19T07:35:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:35:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce11835e-c41d-451a-85d8-a50570f55d5f"

2024-11-19T07:35:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64159dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,33873,-1]
2024-11-19T07:35:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:35:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:35:48,282 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:35:48,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25463823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:35:48,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:48,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:35:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,37281,1732001746728, seqNum=-1]
2024-11-19T07:35:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:35:48,287 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:35:48,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,33873,1732001746459
2024-11-19T07:35:48,290 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:35:48,293 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:35:48,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T07:35:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,33873,1732001746459
2024-11-19T07:35:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b15ea0d
2024-11-19T07:35:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T07:35:48,298 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T07:35:48,299 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T07:35:48,299 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T07:35:48,300 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:35:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:35:48,303 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T07:35:48,303 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,303 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-19T07:35:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:35:48,306 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T07:35:48,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741835_1011 (size=405)
2024-11-19T07:35:48,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741835_1011 (size=405)
2024-11-19T07:35:48,327 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b7e9de47ca6e2be9c1db4d87ee12a0d, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a
2024-11-19T07:35:48,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741836_1012 (size=88)
2024-11-19T07:35:48,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741836_1012 (size=88)
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 7b7e9de47ca6e2be9c1db4d87ee12a0d, disabling compactions & flushes
2024-11-19T07:35:48,334 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d. after waiting 0 ms
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,334 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,334 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:
Waiting for close lock at 1732001748334Disabling compacts and flushes for region at 1732001748334Disabling writes for close at 1732001748334Writing region close event to WAL at 1732001748334Closed at 1732001748334
2024-11-19T07:35:48,335 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T07:35:48,336 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732001748335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001748335"}]},"ts":"1732001748335"}
2024-11-19T07:35:48,338 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T07:35:48,339 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T07:35:48,340 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001748339"}]},"ts":"1732001748339"}
2024-11-19T07:35:48,342 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-19T07:35:48,343 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b7e9de47ca6e2be9c1db4d87ee12a0d, ASSIGN}]
2024-11-19T07:35:48,344 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b7e9de47ca6e2be9c1db4d87ee12a0d, ASSIGN
2024-11-19T07:35:48,345 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b7e9de47ca6e2be9c1db4d87ee12a0d, ASSIGN; state=OFFLINE, location=db0afee3eab9,37281,1732001746728; forceNewPlan=false, retain=false
2024-11-19T07:35:48,496 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b7e9de47ca6e2be9c1db4d87ee12a0d, regionState=OPENING, regionLocation=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:48,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b7e9de47ca6e2be9c1db4d87ee12a0d, ASSIGN because future has completed
2024-11-19T07:35:48,499 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b7e9de47ca6e2be9c1db4d87ee12a0d, server=db0afee3eab9,37281,1732001746728}]
2024-11-19T07:35:48,656 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,656 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b7e9de47ca6e2be9c1db4d87ee12a0d, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:35:48,656 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,656 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:35:48,656 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,656 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,657 INFO  [StoreOpener-7b7e9de47ca6e2be9c1db4d87ee12a0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7b7e9de47ca6e2be9c1db4d87ee12a0d 
2024-11-19T07:35:48,659 INFO  [StoreOpener-7b7e9de47ca6e2be9c1db4d87ee12a0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b7e9de47ca6e2be9c1db4d87ee12a0d columnFamilyName info
2024-11-19T07:35:48,659 DEBUG [StoreOpener-7b7e9de47ca6e2be9c1db4d87ee12a0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:35:48,659 INFO  [StoreOpener-7b7e9de47ca6e2be9c1db4d87ee12a0d-1 {}] regionserver.HStore(327): Store=7b7e9de47ca6e2be9c1db4d87ee12a0d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:35:48,659 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,660 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,660 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,661 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,661 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,663 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,665 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:35:48,666 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7b7e9de47ca6e2be9c1db4d87ee12a0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=878158, jitterRate=0.1166364848613739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:35:48,666 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:35:48,667 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:
Running coprocessor pre-open hook at 1732001748656Writing region info on filesystem at 1732001748656Initializing all the Stores at 1732001748657 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001748657Cleaning up temporary data from old regions at 1732001748661 (+4 ms)Running coprocessor post-open hooks at 1732001748666 (+5 ms)Region opened successfully at 1732001748667 (+1 ms)
2024-11-19T07:35:48,668 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d., pid=6, masterSystemTime=1732001748652
2024-11-19T07:35:48,670 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,671 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:48,671 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b7e9de47ca6e2be9c1db4d87ee12a0d, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,37281,1732001746728
2024-11-19T07:35:48,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b7e9de47ca6e2be9c1db4d87ee12a0d, server=db0afee3eab9,37281,1732001746728 because future has completed
2024-11-19T07:35:48,677 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T07:35:48,677 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b7e9de47ca6e2be9c1db4d87ee12a0d, server=db0afee3eab9,37281,1732001746728 in 176 msec
2024-11-19T07:35:48,680 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T07:35:48,680 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b7e9de47ca6e2be9c1db4d87ee12a0d, ASSIGN in 334 msec
2024-11-19T07:35:48,681 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T07:35:48,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001748681"}]},"ts":"1732001748681"}
2024-11-19T07:35:48,683 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-19T07:35:48,684 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T07:35:48,687 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 384 msec
2024-11-19T07:35:49,104 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:49,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:50,105 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:50,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:51,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:35:51,077 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T07:35:51,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:35:51,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T07:35:51,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:35:51,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T07:35:51,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:35:51,079 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T07:35:51,106 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:51,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:52,106 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:52,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:53,107 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:53,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:53,574 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:35:53,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,578 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,602 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,602 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,602 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,603 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,603 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,603 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,607 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,608 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,608 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,611 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:35:53,617 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:35:53,617 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-19T07:35:54,108 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:54,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:55,108 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:55,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:56,109 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:56,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:57,110 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:57,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:58,111 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:58,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:35:58,318 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T07:35:58,318 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-19T07:35:58,323 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:35:58,323 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d., hostname=db0afee3eab9,37281,1732001746728, seqNum=2]
2024-11-19T07:35:58,337 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:35:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:35:58,344 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T07:35:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-19T07:35:58,345 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T07:35:58,347 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T07:35:58,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37281 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-19T07:35:58,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:58,509 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:35:58,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/22236520868743f8af1dd3ada9627f2f is 1080, key is row0001/info:/1732001758329/Put/seqid=0
2024-11-19T07:35:58,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741837_1013 (size=6033)
2024-11-19T07:35:58,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741837_1013 (size=6033)
2024-11-19T07:35:58,531 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/22236520868743f8af1dd3ada9627f2f
2024-11-19T07:35:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/22236520868743f8af1dd3ada9627f2f as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f
2024-11-19T07:35:58,553 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f, entries=1, sequenceid=5, filesize=5.9 K
2024-11-19T07:35:58,554 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 45ms, sequenceid=5, compaction requested=false
2024-11-19T07:35:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:

2024-11-19T07:35:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:35:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-19T07:35:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-19T07:35:58,562 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-19T07:35:58,562 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 212 msec
2024-11-19T07:35:58,564 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 224 msec
2024-11-19T07:35:59,111 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:35:59,291 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:00,112 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:00,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:01,113 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:01,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:02,113 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:02,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:03,114 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:03,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:04,115 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:04,115 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta after 68058ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:36:04,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:05,116 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:05,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:06,117 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:06,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:07,117 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:07,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:08,118 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:08,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-19T07:36:08,439 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T07:36:08,442 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-19T07:36:08,446 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T07:36:08,447 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T07:36:08,447 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T07:36:08,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37281 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-19T07:36:08,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:08,602 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:36:08,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fdd3c7ea90f74254a7ba4156f6261523 is 1080, key is row0002/info:/1732001768440/Put/seqid=0
2024-11-19T07:36:08,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741838_1014 (size=6033)
2024-11-19T07:36:08,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741838_1014 (size=6033)
2024-11-19T07:36:08,616 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fdd3c7ea90f74254a7ba4156f6261523
2024-11-19T07:36:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fdd3c7ea90f74254a7ba4156f6261523 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523
2024-11-19T07:36:08,631 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523, entries=1, sequenceid=9, filesize=5.9 K
2024-11-19T07:36:08,632 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 31ms, sequenceid=9, compaction requested=false
2024-11-19T07:36:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:

2024-11-19T07:36:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-19T07:36:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-19T07:36:08,636 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-19T07:36:08,636 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-11-19T07:36:08,638 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-11-19T07:36:09,119 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:09,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:09,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802 after 68049ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-19T07:36:10,119 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:10,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:11,120 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:11,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:12,121 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:12,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:13,121 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:13,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:14,122 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:14,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:15,123 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:15,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:16,123 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:16,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:16,440 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:36:17,124 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:17,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:18,125 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:18,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-19T07:36:18,458 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T07:36:18,461 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37281%2C1732001746728.1732001778461
2024-11-19T07:36:18,523 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:18,524 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:18,524 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:18,524 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:18,524 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:18,524 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001747440 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001778461
2024-11-19T07:36:18,525 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39149:39149),(127.0.0.1/127.0.0.1:39355:39355)]
2024-11-19T07:36:18,525 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001747440 is not closed yet, will try archiving it next time
2024-11-19T07:36:18,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741833_1009 (size=5546)
2024-11-19T07:36:18,526 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:18,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741833_1009 (size=5546)
2024-11-19T07:36:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-19T07:36:18,529 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T07:36:18,530 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T07:36:18,531 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T07:36:18,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37281 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-19T07:36:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:18,684 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:36:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/f462b9539e3f404ea0c56251d33a4134 is 1080, key is row0003/info:/1732001778459/Put/seqid=0
2024-11-19T07:36:18,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741840_1016 (size=6033)
2024-11-19T07:36:18,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741840_1016 (size=6033)
2024-11-19T07:36:18,694 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/f462b9539e3f404ea0c56251d33a4134
2024-11-19T07:36:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/f462b9539e3f404ea0c56251d33a4134 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134
2024-11-19T07:36:18,707 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134, entries=1, sequenceid=13, filesize=5.9 K
2024-11-19T07:36:18,709 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 24ms, sequenceid=13, compaction requested=true
2024-11-19T07:36:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:

2024-11-19T07:36:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-19T07:36:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-19T07:36:18,713 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-19T07:36:18,713 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec
2024-11-19T07:36:18,715 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec
2024-11-19T07:36:19,126 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:19,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:20,127 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:20,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:21,128 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:21,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:22,128 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:22,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:23,129 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:23,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:24,130 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:24,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:25,131 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:25,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:26,131 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:26,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:27,132 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:27,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:28,133 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:28,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-19T07:36:28,608 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T07:36:28,609 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:36:28,610 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:36:28,610 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7b7e9de47ca6e2be9c1db4d87ee12a0d/info is initiating minor compaction (all files)
2024-11-19T07:36:28,610 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:36:28,610 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:28,610 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7b7e9de47ca6e2be9c1db4d87ee12a0d/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:28,610 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134] into tmpdir=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp, totalSize=17.7 K
2024-11-19T07:36:28,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 22236520868743f8af1dd3ada9627f2f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732001758329
2024-11-19T07:36:28,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fdd3c7ea90f74254a7ba4156f6261523, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732001768440
2024-11-19T07:36:28,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f462b9539e3f404ea0c56251d33a4134, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732001778459
2024-11-19T07:36:28,627 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7b7e9de47ca6e2be9c1db4d87ee12a0d#info#compaction#47 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:36:28,628 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/d49a7c8ec5b9498c9705705f586fd155 is 1080, key is row0001/info:/1732001758329/Put/seqid=0
2024-11-19T07:36:28,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741841_1017 (size=8296)
2024-11-19T07:36:28,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741841_1017 (size=8296)
2024-11-19T07:36:28,639 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/d49a7c8ec5b9498c9705705f586fd155 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/d49a7c8ec5b9498c9705705f586fd155
2024-11-19T07:36:28,647 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b7e9de47ca6e2be9c1db4d87ee12a0d/info of 7b7e9de47ca6e2be9c1db4d87ee12a0d into d49a7c8ec5b9498c9705705f586fd155(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:36:28,647 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:

2024-11-19T07:36:28,651 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37281%2C1732001746728.1732001788650
2024-11-19T07:36:28,658 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:28,658 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:28,658 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:28,658 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:28,658 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:28,658 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001778461 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001788650
2024-11-19T07:36:28,659 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39355:39355),(127.0.0.1/127.0.0.1:39149:39149)]
2024-11-19T07:36:28,659 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001778461 is not closed yet, will try archiving it next time
2024-11-19T07:36:28,660 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001747440 to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs/db0afee3eab9%2C37281%2C1732001746728.1732001747440
2024-11-19T07:36:28,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741839_1015 (size=2520)
2024-11-19T07:36:28,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741839_1015 (size=2520)
2024-11-19T07:36:28,661 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-19T07:36:28,664 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-19T07:36:28,665 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-19T07:36:28,665 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-19T07:36:28,777 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T07:36:28,777 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T07:36:28,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37281 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-19T07:36:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:28,819 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:36:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/b3ff0e869322453491ec1f845890b11b is 1080, key is row0000/info:/1732001788649/Put/seqid=0
2024-11-19T07:36:28,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741843_1019 (size=6033)
2024-11-19T07:36:28,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741843_1019 (size=6033)
2024-11-19T07:36:28,831 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/b3ff0e869322453491ec1f845890b11b
2024-11-19T07:36:28,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/b3ff0e869322453491ec1f845890b11b as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/b3ff0e869322453491ec1f845890b11b
2024-11-19T07:36:28,844 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/b3ff0e869322453491ec1f845890b11b, entries=1, sequenceid=18, filesize=5.9 K
2024-11-19T07:36:28,845 INFO  [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 26ms, sequenceid=18, compaction requested=false
2024-11-19T07:36:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:

2024-11-19T07:36:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-19T07:36:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-19T07:36:28,850 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-19T07:36:28,850 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-11-19T07:36:28,852 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-11-19T07:36:29,133 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:29,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:30,134 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:30,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:31,134 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:31,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:32,135 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:32,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:33,136 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:33,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:33,656 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 7b7e9de47ca6e2be9c1db4d87ee12a0d, had cached 0 bytes from a total of 14329
2024-11-19T07:36:34,137 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:34,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:35,137 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:35,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:36,138 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:36,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:37,139 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:37,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:38,140 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:38,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-19T07:36:38,718 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-19T07:36:38,721 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37281%2C1732001746728.1732001798721
2024-11-19T07:36:38,727 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,727 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,727 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,728 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,728 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,728 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001788650 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001798721
2024-11-19T07:36:38,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39355:39355),(127.0.0.1/127.0.0.1:39149:39149)]
2024-11-19T07:36:38,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001788650 is not closed yet, will try archiving it next time
2024-11-19T07:36:38,729 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/WALs/db0afee3eab9,37281,1732001746728/db0afee3eab9%2C37281%2C1732001746728.1732001778461 to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs/db0afee3eab9%2C37281%2C1732001746728.1732001778461
2024-11-19T07:36:38,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:36:38,730 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:36:38,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741842_1018 (size=2026)
2024-11-19T07:36:38,730 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:36:38,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:38,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:38,730 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:36:38,730 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:36:38,730 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=336685861, stopped=false
2024-11-19T07:36:38,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741842_1018 (size=2026)
2024-11-19T07:36:38,731 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,33873,1732001746459
2024-11-19T07:36:38,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:36:38,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:36:38,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:38,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:38,770 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:36:38,771 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:36:38,771 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:36:38,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:38,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,37281,1732001746728' *****
2024-11-19T07:36:38,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:36:38,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:36:38,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:36:38,772 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,37281,1732001746728
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:36:38,772 INFO  [RS:0;db0afee3eab9:37281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:37281.
2024-11-19T07:36:38,773 DEBUG [RS:0;db0afee3eab9:37281 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:36:38,773 DEBUG [RS:0;db0afee3eab9:37281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:38,773 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:36:38,773 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:36:38,773 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:36:38,773 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:36:38,773 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b7e9de47ca6e2be9c1db4d87ee12a0d, disabling compactions & flushes
2024-11-19T07:36:38,773 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:38,773 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:38,773 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d. after waiting 0 ms
2024-11-19T07:36:38,773 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:38,773 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-19T07:36:38,780 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-19T07:36:38,780 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7b7e9de47ca6e2be9c1db4d87ee12a0d=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.}
2024-11-19T07:36:38,780 DEBUG [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7b7e9de47ca6e2be9c1db4d87ee12a0d
2024-11-19T07:36:38,780 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:36:38,780 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:36:38,780 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:36:38,780 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:36:38,780 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:36:38,781 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-19T07:36:38,782 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fb98e9d70bca48ec8f11633b3a82abec is 1080, key is row0001/info:/1732001798719/Put/seqid=0
2024-11-19T07:36:38,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741845_1021 (size=6033)
2024-11-19T07:36:38,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741845_1021 (size=6033)
2024-11-19T07:36:38,790 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fb98e9d70bca48ec8f11633b3a82abec
2024-11-19T07:36:38,797 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/info/f3927422ffec4019905a0287c525db39 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d./info:regioninfo/1732001748671/Put/seqid=0
2024-11-19T07:36:38,797 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/.tmp/info/fb98e9d70bca48ec8f11633b3a82abec as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fb98e9d70bca48ec8f11633b3a82abec
2024-11-19T07:36:38,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741846_1022 (size=7308)
2024-11-19T07:36:38,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741846_1022 (size=7308)
2024-11-19T07:36:38,802 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/info/f3927422ffec4019905a0287c525db39
2024-11-19T07:36:38,803 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fb98e9d70bca48ec8f11633b3a82abec, entries=1, sequenceid=22, filesize=5.9 K
2024-11-19T07:36:38,804 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 31ms, sequenceid=22, compaction requested=true
2024-11-19T07:36:38,807 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134] to archive
2024-11-19T07:36:38,808 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:36:38,810 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/22236520868743f8af1dd3ada9627f2f
2024-11-19T07:36:38,811 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523 to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/fdd3c7ea90f74254a7ba4156f6261523
2024-11-19T07:36:38,812 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134 to hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/info/f462b9539e3f404ea0c56251d33a4134
2024-11-19T07:36:38,813 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:33873 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-19T07:36:38,813 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [22236520868743f8af1dd3ada9627f2f=6033, fdd3c7ea90f74254a7ba4156f6261523=6033, f462b9539e3f404ea0c56251d33a4134=6033]
2024-11-19T07:36:38,817 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b7e9de47ca6e2be9c1db4d87ee12a0d/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-19T07:36:38,818 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:38,818 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b7e9de47ca6e2be9c1db4d87ee12a0d:
Waiting for close lock at 1732001798773Running coprocessor pre-close hooks at 1732001798773Disabling compacts and flushes for region at 1732001798773Disabling writes for close at 1732001798773Obtaining lock to block concurrent updates at 1732001798773Preparing flush snapshotting stores in 7b7e9de47ca6e2be9c1db4d87ee12a0d at 1732001798773Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732001798774 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d. at 1732001798775 (+1 ms)Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d/info: creating writer at 1732001798775Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d/info: appending metadata at 1732001798781 (+6 ms)Flushing 7b7e9de47ca6e2be9c1db4d87ee12a0d/info: closing flushed file at 1732001798781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a6db97: reopening flushed file at 1732001798796 (+15 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b7e9de47ca6e2be9c1db4d87ee12a0d in 31ms, sequenceid=22, compaction requested=true at 1732001798804 (+8 ms)Writing region close event to WAL at 1732001798813 (+9 ms)Running coprocessor post-close hooks at 1732001798818 (+5 ms)Closed at 1732001798818
2024-11-19T07:36:38,818 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732001748299.7b7e9de47ca6e2be9c1db4d87ee12a0d.
2024-11-19T07:36:38,824 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/ns/47d5e1e7cd5a460f82e181d44b9029d7 is 43, key is default/ns:d/1732001748156/Put/seqid=0
2024-11-19T07:36:38,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741847_1023 (size=5153)
2024-11-19T07:36:38,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741847_1023 (size=5153)
2024-11-19T07:36:38,832 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/ns/47d5e1e7cd5a460f82e181d44b9029d7
2024-11-19T07:36:38,850 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/table/f5851cde955d4ab8a0a4af2c3b7a3f53 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732001748681/Put/seqid=0
2024-11-19T07:36:38,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741848_1024 (size=5508)
2024-11-19T07:36:38,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741848_1024 (size=5508)
2024-11-19T07:36:38,855 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/table/f5851cde955d4ab8a0a4af2c3b7a3f53
2024-11-19T07:36:38,861 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/info/f3927422ffec4019905a0287c525db39 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/info/f3927422ffec4019905a0287c525db39
2024-11-19T07:36:38,867 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/info/f3927422ffec4019905a0287c525db39, entries=10, sequenceid=11, filesize=7.1 K
2024-11-19T07:36:38,868 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/ns/47d5e1e7cd5a460f82e181d44b9029d7 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/ns/47d5e1e7cd5a460f82e181d44b9029d7
2024-11-19T07:36:38,874 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/ns/47d5e1e7cd5a460f82e181d44b9029d7, entries=2, sequenceid=11, filesize=5.0 K
2024-11-19T07:36:38,875 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/.tmp/table/f5851cde955d4ab8a0a4af2c3b7a3f53 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/table/f5851cde955d4ab8a0a4af2c3b7a3f53
2024-11-19T07:36:38,881 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/table/f5851cde955d4ab8a0a4af2c3b7a3f53, entries=2, sequenceid=11, filesize=5.4 K
2024-11-19T07:36:38,882 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=11, compaction requested=false
2024-11-19T07:36:38,886 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-19T07:36:38,886 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:36:38,886 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:36:38,886 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001798780Running coprocessor pre-close hooks at 1732001798780Disabling compacts and flushes for region at 1732001798780Disabling writes for close at 1732001798780Obtaining lock to block concurrent updates at 1732001798781 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732001798781Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732001798781Flushing stores of hbase:meta,,1.1588230740 at 1732001798781Flushing 1588230740/info: creating writer at 1732001798782 (+1 ms)Flushing 1588230740/info: appending metadata at 1732001798797 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732001798797Flushing 1588230740/ns: creating writer at 1732001798808 (+11 ms)Flushing 1588230740/ns: appending metadata at 1732001798823 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732001798823Flushing 1588230740/table: creating writer at 1732001798837 (+14 ms)Flushing 1588230740/table: appending metadata at 1732001798850 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732001798850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@295c24d2: reopening flushed file at 1732001798860 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2150fb31: reopening flushed file at 1732001798867 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47dd2be0: reopening flushed file at 1732001798874 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=11, compaction requested=false at 1732001798882 (+8 ms)Writing region close event to WAL at 1732001798883 (+1 ms)Running coprocessor post-close hooks at 1732001798886 (+3 ms)Closed at 1732001798886
2024-11-19T07:36:38,887 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:36:38,980 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,37281,1732001746728; all regions closed.
2024-11-19T07:36:38,981 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,981 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,981 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,981 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,981 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741834_1010 (size=3306)
2024-11-19T07:36:38,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741834_1010 (size=3306)
2024-11-19T07:36:38,985 DEBUG [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs
2024-11-19T07:36:38,985 INFO  [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C37281%2C1732001746728.meta:.meta(num 1732001748063)
2024-11-19T07:36:38,985 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,986 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,986 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,986 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,986 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:38,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741844_1020 (size=1252)
2024-11-19T07:36:38,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741844_1020 (size=1252)
2024-11-19T07:36:38,990 DEBUG [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/oldWALs
2024-11-19T07:36:38,990 INFO  [RS:0;db0afee3eab9:37281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C37281%2C1732001746728:(num 1732001798721)
2024-11-19T07:36:38,990 DEBUG [RS:0;db0afee3eab9:37281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:38,990 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:36:38,991 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:36:38,991 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:36:38,991 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:36:38,991 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:36:38,991 INFO  [RS:0;db0afee3eab9:37281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37281
2024-11-19T07:36:39,002 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:36:39,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,37281,1732001746728
2024-11-19T07:36:39,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:36:39,002 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher.
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f768c9049c8@52920242 rejected from java.util.concurrent.ThreadPoolExecutor@17908674[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 13]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?]
	at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?]
	at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4]
2024-11-19T07:36:39,002 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,37281,1732001746728]
2024-11-19T07:36:39,107 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,37281,1732001746728 already deleted, retry=false
2024-11-19T07:36:39,107 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,37281,1732001746728 expired; onlineServers=0
2024-11-19T07:36:39,107 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,33873,1732001746459' *****
2024-11-19T07:36:39,107 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:36:39,107 INFO  [M:0;db0afee3eab9:33873 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:36:39,107 INFO  [M:0;db0afee3eab9:33873 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:36:39,107 DEBUG [M:0;db0afee3eab9:33873 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:36:39,107 DEBUG [M:0;db0afee3eab9:33873 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:36:39,107 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:36:39,107 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001747211 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001747211,5,FailOnTimeoutGroup]
2024-11-19T07:36:39,107 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001747211 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001747211,5,FailOnTimeoutGroup]
2024-11-19T07:36:39,108 INFO  [M:0;db0afee3eab9:33873 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:36:39,108 INFO  [M:0;db0afee3eab9:33873 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:36:39,108 DEBUG [M:0;db0afee3eab9:33873 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:36:39,108 INFO  [M:0;db0afee3eab9:33873 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:36:39,108 INFO  [M:0;db0afee3eab9:33873 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:36:39,108 INFO  [M:0;db0afee3eab9:33873 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:36:39,108 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:36:39,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:36:39,112 INFO  [RS:0;db0afee3eab9:37281 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:36:39,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x101520549b60001, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:36:39,112 INFO  [RS:0;db0afee3eab9:37281 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,37281,1732001746728; zookeeper connection closed.
2024-11-19T07:36:39,113 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@837f8ff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@837f8ff
2024-11-19T07:36:39,113 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:36:39,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:36:39,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:39,118 DEBUG [M:0;db0afee3eab9:33873 {}] zookeeper.ZKUtil(347): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:36:39,118 WARN  [M:0;db0afee3eab9:33873 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:36:39,118 INFO  [M:0;db0afee3eab9:33873 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/.lastflushedseqids
2024-11-19T07:36:39,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741849_1025 (size=130)
2024-11-19T07:36:39,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741849_1025 (size=130)
2024-11-19T07:36:39,126 INFO  [M:0;db0afee3eab9:33873 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:36:39,126 INFO  [M:0;db0afee3eab9:33873 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:36:39,126 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:36:39,126 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:39,126 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:39,126 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:36:39,126 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:39,127 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB
2024-11-19T07:36:39,140 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:39,144 DEBUG [M:0;db0afee3eab9:33873 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b19879cc68243e6ad4c4a60fd12f26c is 82, key is hbase:meta,,1/info:regioninfo/1732001748096/Put/seqid=0
2024-11-19T07:36:39,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741850_1026 (size=5672)
2024-11-19T07:36:39,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741850_1026 (size=5672)
2024-11-19T07:36:39,149 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b19879cc68243e6ad4c4a60fd12f26c
2024-11-19T07:36:39,169 DEBUG [M:0;db0afee3eab9:33873 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b116096d86945f0b42d901c153cc815 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732001748686/Put/seqid=0
2024-11-19T07:36:39,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741851_1027 (size=7824)
2024-11-19T07:36:39,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741851_1027 (size=7824)
2024-11-19T07:36:39,174 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b116096d86945f0b42d901c153cc815
2024-11-19T07:36:39,178 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b116096d86945f0b42d901c153cc815
2024-11-19T07:36:39,193 DEBUG [M:0;db0afee3eab9:33873 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/add4aef231284997840bc0920743a2f1 is 69, key is db0afee3eab9,37281,1732001746728/rs:state/1732001747235/Put/seqid=0
2024-11-19T07:36:39,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741852_1028 (size=5156)
2024-11-19T07:36:39,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741852_1028 (size=5156)
2024-11-19T07:36:39,199 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/add4aef231284997840bc0920743a2f1
2024-11-19T07:36:39,220 DEBUG [M:0;db0afee3eab9:33873 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e5efb07d247f486cb99af2ee9266167b is 52, key is load_balancer_on/state:d/1732001748292/Put/seqid=0
2024-11-19T07:36:39,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741853_1029 (size=5056)
2024-11-19T07:36:39,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741853_1029 (size=5056)
2024-11-19T07:36:39,225 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e5efb07d247f486cb99af2ee9266167b
2024-11-19T07:36:39,230 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b19879cc68243e6ad4c4a60fd12f26c as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b19879cc68243e6ad4c4a60fd12f26c
2024-11-19T07:36:39,235 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b19879cc68243e6ad4c4a60fd12f26c, entries=8, sequenceid=121, filesize=5.5 K
2024-11-19T07:36:39,236 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b116096d86945f0b42d901c153cc815 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9b116096d86945f0b42d901c153cc815
2024-11-19T07:36:39,241 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b116096d86945f0b42d901c153cc815
2024-11-19T07:36:39,242 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9b116096d86945f0b42d901c153cc815, entries=14, sequenceid=121, filesize=7.6 K
2024-11-19T07:36:39,243 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/add4aef231284997840bc0920743a2f1 as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/add4aef231284997840bc0920743a2f1
2024-11-19T07:36:39,249 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/add4aef231284997840bc0920743a2f1, entries=1, sequenceid=121, filesize=5.0 K
2024-11-19T07:36:39,250 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e5efb07d247f486cb99af2ee9266167b as hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e5efb07d247f486cb99af2ee9266167b
2024-11-19T07:36:39,256 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44151/user/jenkins/test-data/a7bc3a96-659d-685a-eac6-72bce85a9d4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e5efb07d247f486cb99af2ee9266167b, entries=1, sequenceid=121, filesize=4.9 K
2024-11-19T07:36:39,257 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=121, compaction requested=false
2024-11-19T07:36:39,258 INFO  [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:39,258 DEBUG [M:0;db0afee3eab9:33873 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001799126Disabling compacts and flushes for region at 1732001799126Disabling writes for close at 1732001799126Obtaining lock to block concurrent updates at 1732001799127 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001799127Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44647, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1732001799127Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001799128 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001799128Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001799144 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001799144Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001799154 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001799168 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001799168Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001799178 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001799193 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001799193Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001799204 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001799220 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001799220Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f56e167: reopening flushed file at 1732001799229 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7788fcd: reopening flushed file at 1732001799235 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ae85c30: reopening flushed file at 1732001799242 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37bda874: reopening flushed file at 1732001799249 (+7 ms)Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=121, compaction requested=false at 1732001799257 (+8 ms)Writing region close event to WAL at 1732001799258 (+1 ms)Closed at 1732001799258
2024-11-19T07:36:39,258 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:39,259 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:39,259 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:39,259 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:39,259 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:36:39,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45523 is added to blk_1073741830_1006 (size=53044)
2024-11-19T07:36:39,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36715 is added to blk_1073741830_1006 (size=53044)
2024-11-19T07:36:39,261 INFO  [M:0;db0afee3eab9:33873 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:36:39,261 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:36:39,262 INFO  [M:0;db0afee3eab9:33873 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33873
2024-11-19T07:36:39,262 INFO  [M:0;db0afee3eab9:33873 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:36:39,290 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:36:39,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:39,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:36:39,372 INFO  [M:0;db0afee3eab9:33873 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:36:39,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33873-0x101520549b60000, quorum=127.0.0.1:62849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:36:39,397 WARN  [BP-1081112191-172.17.0.2-1732001744154 heartbeating to localhost/127.0.0.1:44151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1081112191-172.17.0.2-1732001744154 (Datanode Uuid ce418414-4e71-4f93-ab4a-eb5bd8f4ca4d) service to localhost/127.0.0.1:44151
2024-11-19T07:36:39,404 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data3/current/BP-1081112191-172.17.0.2-1732001744154 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:36:39,405 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data4/current/BP-1081112191-172.17.0.2-1732001744154 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:36:39,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fc2e521{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:36:39,406 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bff2115{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:36:39,406 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:36:39,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36f52998{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:36:39,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38d3f6f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,STOPPED}
2024-11-19T07:36:39,408 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:36:39,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37d74326{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:36:39,410 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@77f859f0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:36:39,410 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:36:39,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@261a9e0a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:36:39,411 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6c1f86{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,STOPPED}
2024-11-19T07:36:39,412 WARN  [BP-1081112191-172.17.0.2-1732001744154 heartbeating to localhost/127.0.0.1:44151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:36:39,412 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:36:39,412 WARN  [BP-1081112191-172.17.0.2-1732001744154 heartbeating to localhost/127.0.0.1:44151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1081112191-172.17.0.2-1732001744154 (Datanode Uuid 58c7e4f9-f099-45ac-ac72-cf614bb91207) service to localhost/127.0.0.1:44151
2024-11-19T07:36:39,412 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:36:39,413 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data1/current/BP-1081112191-172.17.0.2-1732001744154 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:36:39,413 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/cluster_120cd80b-14dd-c0bb-ed3f-bab97c961119/data/data2/current/BP-1081112191-172.17.0.2-1732001744154 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:36:39,413 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:36:39,419 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fc981fd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:36:39,420 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4732430a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:36:39,420 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:36:39,420 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c020752{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:36:39,420 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77877788{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir/,STOPPED}
2024-11-19T07:36:39,426 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:36:39,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:36:39,457 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 181)
Potentially hanging thread: LeaseRenewer:jenkins@localhost:44151
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44151 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Command processor
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44151
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44151 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44151
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44151 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44151
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:44151
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=11233 (was 11343)
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=11233
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.log.dir so I do NOT create it in target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af30d22-f2a8-a610-0a9c-d8b76c501260/hadoop.tmp.dir so I do NOT create it in target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c, deleteOnExit=true
2024-11-19T07:36:39,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/test.cache.data in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:36:39,465 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:36:39,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:36:39,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:36:39,479 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:36:39,853 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:36:39,858 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:36:39,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:36:39,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:36:39,860 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:36:39,860 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:36:39,861 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75966949{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:36:39,861 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3def21d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:36:39,962 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4328dba6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/java.io.tmpdir/jetty-localhost-43799-hadoop-hdfs-3_4_1-tests_jar-_-any-3876503966952349749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:36:39,962 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1dab95de{HTTP/1.1, (http/1.1)}{localhost:43799}
2024-11-19T07:36:39,962 INFO  [Time-limited test {}] server.Server(415): Started @256651ms
2024-11-19T07:36:39,975 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:36:40,141 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:40,275 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:36:40,278 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:36:40,280 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:36:40,280 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:36:40,280 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:36:40,280 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60b9b83d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:36:40,281 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b95b0ea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:36:40,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:40,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77f3cd08{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/java.io.tmpdir/jetty-localhost-42441-hadoop-hdfs-3_4_1-tests_jar-_-any-10378095080896032510/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:36:40,385 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2401b6df{HTTP/1.1, (http/1.1)}{localhost:42441}
2024-11-19T07:36:40,385 INFO  [Time-limited test {}] server.Server(415): Started @257074ms
2024-11-19T07:36:40,386 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:36:40,414 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:36:40,416 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:36:40,417 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:36:40,417 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:36:40,417 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:36:40,417 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fc2e7d1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:36:40,418 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dea9c62{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:36:40,522 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ced388d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/java.io.tmpdir/jetty-localhost-34639-hadoop-hdfs-3_4_1-tests_jar-_-any-4524304575209076531/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:36:40,523 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30ce08b8{HTTP/1.1, (http/1.1)}{localhost:34639}
2024-11-19T07:36:40,523 INFO  [Time-limited test {}] server.Server(415): Started @257212ms
2024-11-19T07:36:40,526 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:36:41,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:36:41,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:36:41,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:36:41,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-19T07:36:41,141 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:41,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:41,556 WARN  [Thread-1989 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data1/current/BP-2122081010-172.17.0.2-1732001799482/current, will proceed with Du for space computation calculation, 
2024-11-19T07:36:41,557 WARN  [Thread-1990 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data2/current/BP-2122081010-172.17.0.2-1732001799482/current, will proceed with Du for space computation calculation, 
2024-11-19T07:36:41,575 WARN  [Thread-1953 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:36:41,577 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1f39315bb5f5711 with lease ID 0x4d63f58138c92b83: Processing first storage report for DS-01782e97-5562-4997-b9d3-16ea0a11faee from datanode DatanodeRegistration(127.0.0.1:35161, datanodeUuid=05167d0f-86cf-420d-9838-091778010c66, infoPort=33437, infoSecurePort=0, ipcPort=42885, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482)
2024-11-19T07:36:41,577 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1f39315bb5f5711 with lease ID 0x4d63f58138c92b83: from storage DS-01782e97-5562-4997-b9d3-16ea0a11faee node DatanodeRegistration(127.0.0.1:35161, datanodeUuid=05167d0f-86cf-420d-9838-091778010c66, infoPort=33437, infoSecurePort=0, ipcPort=42885, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:36:41,577 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1f39315bb5f5711 with lease ID 0x4d63f58138c92b83: Processing first storage report for DS-95dbb983-c8b9-49c4-b48f-27dc53dbd253 from datanode DatanodeRegistration(127.0.0.1:35161, datanodeUuid=05167d0f-86cf-420d-9838-091778010c66, infoPort=33437, infoSecurePort=0, ipcPort=42885, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482)
2024-11-19T07:36:41,577 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1f39315bb5f5711 with lease ID 0x4d63f58138c92b83: from storage DS-95dbb983-c8b9-49c4-b48f-27dc53dbd253 node DatanodeRegistration(127.0.0.1:35161, datanodeUuid=05167d0f-86cf-420d-9838-091778010c66, infoPort=33437, infoSecurePort=0, ipcPort=42885, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:36:41,676 WARN  [Thread-2000 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data3/current/BP-2122081010-172.17.0.2-1732001799482/current, will proceed with Du for space computation calculation, 
2024-11-19T07:36:41,676 WARN  [Thread-2001 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data4/current/BP-2122081010-172.17.0.2-1732001799482/current, will proceed with Du for space computation calculation, 
2024-11-19T07:36:41,693 WARN  [Thread-1976 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:36:41,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x67f30627120b9ef with lease ID 0x4d63f58138c92b84: Processing first storage report for DS-72bf198b-1055-4074-9610-7fa95161553c from datanode DatanodeRegistration(127.0.0.1:34561, datanodeUuid=edf25135-b238-48f5-8014-b4eca792342e, infoPort=42025, infoSecurePort=0, ipcPort=38663, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482)
2024-11-19T07:36:41,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67f30627120b9ef with lease ID 0x4d63f58138c92b84: from storage DS-72bf198b-1055-4074-9610-7fa95161553c node DatanodeRegistration(127.0.0.1:34561, datanodeUuid=edf25135-b238-48f5-8014-b4eca792342e, infoPort=42025, infoSecurePort=0, ipcPort=38663, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:36:41,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x67f30627120b9ef with lease ID 0x4d63f58138c92b84: Processing first storage report for DS-86865b9a-fd2b-4493-b122-b0b95e20582f from datanode DatanodeRegistration(127.0.0.1:34561, datanodeUuid=edf25135-b238-48f5-8014-b4eca792342e, infoPort=42025, infoSecurePort=0, ipcPort=38663, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482)
2024-11-19T07:36:41,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67f30627120b9ef with lease ID 0x4d63f58138c92b84: from storage DS-86865b9a-fd2b-4493-b122-b0b95e20582f node DatanodeRegistration(127.0.0.1:34561, datanodeUuid=edf25135-b238-48f5-8014-b4eca792342e, infoPort=42025, infoSecurePort=0, ipcPort=38663, storageInfo=lv=-57;cid=testClusterID;nsid=1762741141;c=1732001799482), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:36:41,755 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc
2024-11-19T07:36:41,758 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/zookeeper_0, clientPort=56379, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:36:41,758 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56379
2024-11-19T07:36:41,759 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,760 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:36:41,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:36:41,770 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74 with version=8
2024-11-19T07:36:41,770 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:36:41,772 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:36:41,773 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37997
2024-11-19T07:36:41,775 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37997 connecting to ZooKeeper ensemble=127.0.0.1:56379
2024-11-19T07:36:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379970x0, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:36:41,824 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37997-0x101520621c80000 connected
2024-11-19T07:36:41,907 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,909 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:36:41,912 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74, hbase.cluster.distributed=false
2024-11-19T07:36:41,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:36:41,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37997
2024-11-19T07:36:41,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37997
2024-11-19T07:36:41,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37997
2024-11-19T07:36:41,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37997
2024-11-19T07:36:41,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37997
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:36:41,934 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:36:41,935 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36609
2024-11-19T07:36:41,936 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36609 connecting to ZooKeeper ensemble=127.0.0.1:56379
2024-11-19T07:36:41,937 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,938 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:41,949 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366090x0, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:36:41,949 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366090x0, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:36:41,949 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36609-0x101520621c80001 connected
2024-11-19T07:36:41,950 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:36:41,950 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:36:41,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:36:41,951 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:36:41,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36609
2024-11-19T07:36:41,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36609
2024-11-19T07:36:41,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36609
2024-11-19T07:36:41,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36609
2024-11-19T07:36:41,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36609
2024-11-19T07:36:41,971 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:37997
2024-11-19T07:36:41,972 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,37997,1732001801772
2024-11-19T07:36:41,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:36:41,980 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:36:41,981 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,37997,1732001801772
2024-11-19T07:36:41,991 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:36:41,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:41,991 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:41,992 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:36:41,992 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,37997,1732001801772 from backup master directory
2024-11-19T07:36:42,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,37997,1732001801772
2024-11-19T07:36:42,001 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:36:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:36:42,002 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:36:42,002 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,37997,1732001801772
2024-11-19T07:36:42,006 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/hbase.id] with ID: bc1a89d7-443c-46ff-bdeb-bfbbc45f1aba
2024-11-19T07:36:42,007 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/.tmp/hbase.id
2024-11-19T07:36:42,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:36:42,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:36:42,015 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/.tmp/hbase.id]:[hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/hbase.id]
2024-11-19T07:36:42,027 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:42,027 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:36:42,029 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T07:36:42,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,040 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:36:42,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:36:42,046 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:36:42,047 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:36:42,047 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:36:42,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:36:42,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:36:42,055 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store
2024-11-19T07:36:42,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:36:42,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:36:42,061 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:42,062 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:36:42,062 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:42,062 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:42,062 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:36:42,062 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:42,062 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:36:42,062 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001802062Disabling compacts and flushes for region at 1732001802062Disabling writes for close at 1732001802062Writing region close event to WAL at 1732001802062Closed at 1732001802062
2024-11-19T07:36:42,065 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/.initializing
2024-11-19T07:36:42,065 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/WALs/db0afee3eab9,37997,1732001801772
2024-11-19T07:36:42,067 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C37997%2C1732001801772, suffix=, logDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/WALs/db0afee3eab9,37997,1732001801772, archiveDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/oldWALs, maxLogs=10
2024-11-19T07:36:42,068 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C37997%2C1732001801772.1732001802068
2024-11-19T07:36:42,073 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/WALs/db0afee3eab9,37997,1732001801772/db0afee3eab9%2C37997%2C1732001801772.1732001802068
2024-11-19T07:36:42,073 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42025:42025),(127.0.0.1/127.0.0.1:33437:33437)]
2024-11-19T07:36:42,074 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:36:42,074 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:42,074 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,074 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,076 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:36:42,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:36:42,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:42,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:36:42,078 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:36:42,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,079 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:36:42,079 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:36:42,080 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:36:42,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,080 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:36:42,081 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:36:42,081 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:36:42,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,082 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:36:42,082 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,083 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,083 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,084 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,084 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,084 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:36:42,085 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:36:42,087 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:36:42,087 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707569, jitterRate=-0.10028053820133209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:36:42,088 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001802074Initializing all the Stores at 1732001802075 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001802075Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001802075Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001802075Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001802075Cleaning up temporary data from old regions at 1732001802084 (+9 ms)Region opened successfully at 1732001802088 (+4 ms)
2024-11-19T07:36:42,088 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:36:42,091 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2015349e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:36:42,092 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:36:42,092 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:36:42,092 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:36:42,092 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:36:42,093 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:36:42,093 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:36:42,093 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:36:42,095 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:36:42,096 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:36:42,107 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:36:42,107 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:36:42,108 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:36:42,117 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:36:42,118 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:36:42,119 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:36:42,128 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:36:42,129 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:36:42,138 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:36:42,140 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:36:42,142 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:42,149 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:36:42,159 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:36:42,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:36:42,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,159 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,160 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,37997,1732001801772, sessionid=0x101520621c80000, setting cluster-up flag (Was=false)
2024-11-19T07:36:42,180 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,212 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:36:42,213 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,37997,1732001801772
2024-11-19T07:36:42,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,361 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,408 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:36:42,409 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,37997,1732001801772
2024-11-19T07:36:42,410 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:36:42,411 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:36:42,412 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:36:42,412 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:36:42,412 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,37997,1732001801772
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:36:42,413 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:36:42,413 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:36:42,413 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:36:42,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:36:42,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:36:42,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:36:42,414 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001832417
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:36:42,417 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:36:42,418 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,418 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:36:42,418 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:36:42,418 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:36:42,418 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:36:42,418 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:36:42,418 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:36:42,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:36:42,419 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001802419,5,FailOnTimeoutGroup]
2024-11-19T07:36:42,419 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001802419,5,FailOnTimeoutGroup]
2024-11-19T07:36:42,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:36:42,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,419 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,419 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,419 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:36:42,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:36:42,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:36:42,427 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:36:42,427 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74
2024-11-19T07:36:42,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:36:42,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:36:42,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:42,434 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:36:42,436 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:36:42,436 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,436 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:42,436 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:36:42,437 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:36:42,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:42,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:36:42,439 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:36:42,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:42,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:36:42,441 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:36:42,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:42,441 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:42,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:36:42,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740
2024-11-19T07:36:42,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740
2024-11-19T07:36:42,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:36:42,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:36:42,444 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:36:42,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:36:42,447 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:36:42,447 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=710083, jitterRate=-0.09708377718925476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001802433Initializing all the Stores at 1732001802434 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001802434Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001802434Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001802434Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001802434Cleaning up temporary data from old regions at 1732001802443 (+9 ms)Region opened successfully at 1732001802448 (+5 ms)
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:36:42,448 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:36:42,448 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:36:42,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001802448Disabling compacts and flushes for region at 1732001802448Disabling writes for close at 1732001802448Writing region close event to WAL at 1732001802448Closed at 1732001802448
2024-11-19T07:36:42,449 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:36:42,449 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:36:42,450 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:36:42,451 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:36:42,452 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:36:42,461 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(746): ClusterId : bc1a89d7-443c-46ff-bdeb-bfbbc45f1aba
2024-11-19T07:36:42,461 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:36:42,472 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:36:42,472 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:36:42,483 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:36:42,483 DEBUG [RS:0;db0afee3eab9:36609 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd9e4b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:36:42,500 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:36609
2024-11-19T07:36:42,500 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:36:42,500 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:36:42,500 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:36:42,501 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,37997,1732001801772 with port=36609, startcode=1732001801933
2024-11-19T07:36:42,501 DEBUG [RS:0;db0afee3eab9:36609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:36:42,503 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36505, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:36:42,504 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37997 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,504 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37997 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,505 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74
2024-11-19T07:36:42,505 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36347
2024-11-19T07:36:42,505 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:36:42,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:36:42,518 DEBUG [RS:0;db0afee3eab9:36609 {}] zookeeper.ZKUtil(111): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,518 WARN  [RS:0;db0afee3eab9:36609 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:36:42,518 INFO  [RS:0;db0afee3eab9:36609 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:36:42,518 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,518 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,36609,1732001801933]
2024-11-19T07:36:42,521 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:36:42,523 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:36:42,524 INFO  [RS:0;db0afee3eab9:36609 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:36:42,524 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,524 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:36:42,525 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:36:42,525 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,525 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,526 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,526 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:36:42,526 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:36:42,526 DEBUG [RS:0;db0afee3eab9:36609 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,526 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,36609,1732001801933-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:36:42,542 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:36:42,542 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,36609,1732001801933-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,542 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,542 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.Replication(171): db0afee3eab9,36609,1732001801933 started
2024-11-19T07:36:42,556 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:42,556 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,36609,1732001801933, RpcServer on db0afee3eab9/172.17.0.2:36609, sessionid=0x101520621c80001
2024-11-19T07:36:42,556 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:36:42,556 DEBUG [RS:0;db0afee3eab9:36609 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,556 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,36609,1732001801933'
2024-11-19T07:36:42,556 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,36609,1732001801933'
2024-11-19T07:36:42,557 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:36:42,558 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:36:42,558 DEBUG [RS:0;db0afee3eab9:36609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:36:42,558 INFO  [RS:0;db0afee3eab9:36609 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:36:42,558 INFO  [RS:0;db0afee3eab9:36609 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:36:42,602 WARN  [db0afee3eab9:37997 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:36:42,661 INFO  [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C36609%2C1732001801933, suffix=, logDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933, archiveDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs, maxLogs=32
2024-11-19T07:36:42,661 INFO  [RS:0;db0afee3eab9:36609 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C36609%2C1732001801933.1732001802661
2024-11-19T07:36:42,669 INFO  [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001802661
2024-11-19T07:36:42,677 DEBUG [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33437:33437),(127.0.0.1/127.0.0.1:42025:42025)]
2024-11-19T07:36:42,852 DEBUG [db0afee3eab9:37997 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:36:42,853 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:42,854 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,36609,1732001801933, state=OPENING
2024-11-19T07:36:42,886 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:36:42,980 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:36:42,981 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:36:42,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:36:42,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:36:42,982 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,36609,1732001801933}]
2024-11-19T07:36:43,136 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:36:43,138 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58599, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:36:43,142 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:36:43,142 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:36:43,142 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:43,144 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C36609%2C1732001801933.meta, suffix=.meta, logDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933, archiveDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs, maxLogs=32
2024-11-19T07:36:43,145 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C36609%2C1732001801933.meta.1732001803145.meta
2024-11-19T07:36:43,150 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.meta.1732001803145.meta
2024-11-19T07:36:43,151 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33437:33437),(127.0.0.1/127.0.0.1:42025:42025)]
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:36:43,152 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:36:43,152 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:36:43,154 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:36:43,154 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:36:43,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,155 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:43,155 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:36:43,155 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:36:43,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,156 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:43,156 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:36:43,156 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:36:43,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,157 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:43,157 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:36:43,157 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:36:43,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,158 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:36:43,158 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:36:43,159 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740
2024-11-19T07:36:43,160 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740
2024-11-19T07:36:43,161 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:36:43,161 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:36:43,162 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:36:43,163 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:36:43,164 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=803013, jitterRate=0.021084487438201904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:36:43,164 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:36:43,165 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001803153Writing region info on filesystem at 1732001803153Initializing all the Stores at 1732001803153Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001803153Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001803154 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001803154Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001803154Cleaning up temporary data from old regions at 1732001803161 (+7 ms)Running coprocessor post-open hooks at 1732001803164 (+3 ms)Region opened successfully at 1732001803165 (+1 ms)
2024-11-19T07:36:43,166 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001803135
2024-11-19T07:36:43,168 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:36:43,168 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:36:43,169 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:43,169 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,36609,1732001801933, state=OPEN
2024-11-19T07:36:43,210 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:36:43,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:36:43,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:36:43,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:36:43,210 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:43,215 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:36:43,215 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,36609,1732001801933 in 229 msec
2024-11-19T07:36:43,218 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:36:43,218 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 765 msec
2024-11-19T07:36:43,219 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:36:43,219 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:36:43,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:36:43,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,36609,1732001801933, seqNum=-1]
2024-11-19T07:36:43,220 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:36:43,222 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36091, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:36:43,228 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 816 msec
2024-11-19T07:36:43,228 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001803228, completionTime=-1
2024-11-19T07:36:43,228 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:36:43,228 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:36:43,230 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:36:43,230 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001863230
2024-11-19T07:36:43,230 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001923230
2024-11-19T07:36:43,230 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-19T07:36:43,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:37997, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,231 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,232 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,233 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.233sec
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:36:43,235 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:36:43,238 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:36:43,238 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:36:43,238 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,37997,1732001801772-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:36:43,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a437f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:36:43,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,37997,-1 for getting cluster id
2024-11-19T07:36:43,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:36:43,264 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bc1a89d7-443c-46ff-bdeb-bfbbc45f1aba'
2024-11-19T07:36:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:36:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bc1a89d7-443c-46ff-bdeb-bfbbc45f1aba"

2024-11-19T07:36:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ba3a4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:36:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,37997,-1]
2024-11-19T07:36:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:36:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:36:43,266 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:36:43,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49b7a587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:36:43,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:36:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,36609,1732001801933, seqNum=-1]
2024-11-19T07:36:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:36:43,271 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:36:43,273 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,37997,1732001801772
2024-11-19T07:36:43,273 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:36:43,276 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:36:43,276 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-19T07:36:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is db0afee3eab9,37997,1732001801772
2024-11-19T07:36:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35beb78b
2024-11-19T07:36:43,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-19T07:36:43,279 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-19T07:36:43,279 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-19T07:36:43,279 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-19T07:36:43,280 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:36:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-19T07:36:43,282 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-19T07:36:43,282 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,283 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-19T07:36:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:36:43,284 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-19T07:36:43,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741835_1011 (size=381)
2024-11-19T07:36:43,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741835_1011 (size=381)
2024-11-19T07:36:43,294 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5858b23891076d8cb7718291e3b42567, NAME => 'TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74
2024-11-19T07:36:43,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741836_1012 (size=64)
2024-11-19T07:36:43,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741836_1012 (size=64)
2024-11-19T07:36:43,300 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:43,300 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5858b23891076d8cb7718291e3b42567, disabling compactions & flushes
2024-11-19T07:36:43,300 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,300 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,300 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567. after waiting 0 ms
2024-11-19T07:36:43,301 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,301 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,301 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5858b23891076d8cb7718291e3b42567:
Waiting for close lock at 1732001803300Disabling compacts and flushes for region at 1732001803300Disabling writes for close at 1732001803300Writing region close event to WAL at 1732001803301 (+1 ms)Closed at 1732001803301
2024-11-19T07:36:43,302 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-19T07:36:43,302 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732001803302"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001803302"}]},"ts":"1732001803302"}
2024-11-19T07:36:43,304 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-19T07:36:43,305 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-19T07:36:43,306 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001803306"}]},"ts":"1732001803306"}
2024-11-19T07:36:43,308 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-19T07:36:43,308 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, ASSIGN}]
2024-11-19T07:36:43,310 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, ASSIGN
2024-11-19T07:36:43,311 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, ASSIGN; state=OFFLINE, location=db0afee3eab9,36609,1732001801933; forceNewPlan=false, retain=false
2024-11-19T07:36:43,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:43,462 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5858b23891076d8cb7718291e3b42567, regionState=OPENING, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:43,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, ASSIGN because future has completed
2024-11-19T07:36:43,465 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933}]
2024-11-19T07:36:43,624 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,624 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5858b23891076d8cb7718291e3b42567, NAME => 'TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:36:43,625 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,625 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:36:43,625 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,625 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,626 INFO  [StoreOpener-5858b23891076d8cb7718291e3b42567-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5858b23891076d8cb7718291e3b42567 
2024-11-19T07:36:43,628 INFO  [StoreOpener-5858b23891076d8cb7718291e3b42567-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5858b23891076d8cb7718291e3b42567 columnFamilyName info
2024-11-19T07:36:43,628 DEBUG [StoreOpener-5858b23891076d8cb7718291e3b42567-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:36:43,628 INFO  [StoreOpener-5858b23891076d8cb7718291e3b42567-1 {}] regionserver.HStore(327): Store=5858b23891076d8cb7718291e3b42567/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:36:43,628 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,629 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,629 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,630 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,630 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,632 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,634 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:36:43,634 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5858b23891076d8cb7718291e3b42567; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794451, jitterRate=0.010197415947914124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:36:43,634 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:43,635 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5858b23891076d8cb7718291e3b42567:
Running coprocessor pre-open hook at 1732001803625Writing region info on filesystem at 1732001803625Initializing all the Stores at 1732001803626 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001803626Cleaning up temporary data from old regions at 1732001803630 (+4 ms)Running coprocessor post-open hooks at 1732001803634 (+4 ms)Region opened successfully at 1732001803635 (+1 ms)
2024-11-19T07:36:43,636 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., pid=6, masterSystemTime=1732001803618
2024-11-19T07:36:43,638 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,638 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:43,639 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5858b23891076d8cb7718291e3b42567, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:43,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933 because future has completed
2024-11-19T07:36:43,645 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-19T07:36:43,645 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933 in 178 msec
2024-11-19T07:36:43,647 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-19T07:36:43,648 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, ASSIGN in 337 msec
2024-11-19T07:36:43,648 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-19T07:36:43,649 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732001803648"}]},"ts":"1732001803648"}
2024-11-19T07:36:43,651 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-19T07:36:43,652 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-19T07:36:43,654 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 372 msec
2024-11-19T07:36:43,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,842 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,842 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,842 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:43,845 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,143 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:44,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:44,348 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:36:44,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,350 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:44,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:36:45,144 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:45,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:46,144 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:46,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:47,145 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:47,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:48,146 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:48,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:48,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:36:48,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-19T07:36:49,147 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:49,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:50,148 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:50,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:51,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:36:51,077 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-19T07:36:51,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:36:51,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-19T07:36:51,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:36:51,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-19T07:36:51,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-19T07:36:51,079 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-19T07:36:51,148 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:51,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:52,149 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:52,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:53,150 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:53,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-19T07:36:53,378 INFO  [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-19T07:36:53,378 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-19T07:36:53,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-19T07:36:53,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2]
2024-11-19T07:36:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:53,398 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5858b23891076d8cb7718291e3b42567 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:36:53,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/8412e0dd69b14d7fb89ec26f06e11906 is 1080, key is row0001/info:/1732001813385/Put/seqid=0
2024-11-19T07:36:53,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741837_1013 (size=12509)
2024-11-19T07:36:53,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741837_1013 (size=12509)
2024-11-19T07:36:53,421 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/8412e0dd69b14d7fb89ec26f06e11906
2024-11-19T07:36:53,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/8412e0dd69b14d7fb89ec26f06e11906 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906
2024-11-19T07:36:53,434 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906, entries=7, sequenceid=11, filesize=12.2 K
2024-11-19T07:36:53,435 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for 5858b23891076d8cb7718291e3b42567 in 37ms, sequenceid=11, compaction requested=false
2024-11-19T07:36:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:36:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:53,437 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5858b23891076d8cb7718291e3b42567 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB
2024-11-19T07:36:53,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/057fbb019e994407b37c9b08dd76d606 is 1080, key is row0008/info:/1732001813400/Put/seqid=0
2024-11-19T07:36:53,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741838_1014 (size=26530)
2024-11-19T07:36:53,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741838_1014 (size=26530)
2024-11-19T07:36:53,446 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/057fbb019e994407b37c9b08dd76d606
2024-11-19T07:36:53,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/057fbb019e994407b37c9b08dd76d606 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606
2024-11-19T07:36:53,456 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606, entries=20, sequenceid=34, filesize=25.9 K
2024-11-19T07:36:53,457 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for 5858b23891076d8cb7718291e3b42567 in 20ms, sequenceid=34, compaction requested=false
2024-11-19T07:36:53,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:36:53,457 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K
2024-11-19T07:36:53,457 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:53,457 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606 because midkey is the same as first or last row
2024-11-19T07:36:54,150 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:54,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:55,151 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:55,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:55,453 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5858b23891076d8cb7718291e3b42567 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:36:55,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/627b4d117a5f41cfb8986e773afb2c69 is 1080, key is row0028/info:/1732001813438/Put/seqid=0
2024-11-19T07:36:55,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741839_1015 (size=12509)
2024-11-19T07:36:55,468 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/627b4d117a5f41cfb8986e773afb2c69
2024-11-19T07:36:55,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741839_1015 (size=12509)
2024-11-19T07:36:55,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/627b4d117a5f41cfb8986e773afb2c69 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69
2024-11-19T07:36:55,483 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69, entries=7, sequenceid=44, filesize=12.2 K
2024-11-19T07:36:55,484 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 5858b23891076d8cb7718291e3b42567 in 31ms, sequenceid=44, compaction requested=true
2024-11-19T07:36:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:36:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K
2024-11-19T07:36:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606 because midkey is the same as first or last row
2024-11-19T07:36:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5858b23891076d8cb7718291e3b42567:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:36:55,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:36:55,485 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:36:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 5858b23891076d8cb7718291e3b42567
2024-11-19T07:36:55,486 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5858b23891076d8cb7718291e3b42567 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-19T07:36:55,486 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:36:55,486 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): 5858b23891076d8cb7718291e3b42567/info is initiating minor compaction (all files)
2024-11-19T07:36:55,486 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5858b23891076d8cb7718291e3b42567/info in TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:36:55,486 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp, totalSize=50.3 K
2024-11-19T07:36:55,487 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8412e0dd69b14d7fb89ec26f06e11906, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732001813385
2024-11-19T07:36:55,488 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057fbb019e994407b37c9b08dd76d606, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732001813400
2024-11-19T07:36:55,489 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627b4d117a5f41cfb8986e773afb2c69, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732001813438
2024-11-19T07:36:55,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/9c7214f34a9f4956baa80ef6d5c4f4ca is 1080, key is row0035/info:/1732001815454/Put/seqid=0
2024-11-19T07:36:55,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741840_1016 (size=18987)
2024-11-19T07:36:55,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741840_1016 (size=18987)
2024-11-19T07:36:55,515 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5858b23891076d8cb7718291e3b42567#info#compaction#61 average throughput is 17.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:36:55,515 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/50ffea9526ef4345bb38edf524b0f838 is 1080, key is row0001/info:/1732001813385/Put/seqid=0
2024-11-19T07:36:55,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741841_1017 (size=41747)
2024-11-19T07:36:55,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741841_1017 (size=41747)
2024-11-19T07:36:55,539 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/50ffea9526ef4345bb38edf524b0f838 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838
2024-11-19T07:36:55,545 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-19T07:36:55,549 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5858b23891076d8cb7718291e3b42567/info of 5858b23891076d8cb7718291e3b42567 into 50ffea9526ef4345bb38edf524b0f838(size=40.8 K), total size for store is 40.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:36:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36538 deadline: 1732001825544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933
2024-11-19T07:36:55,549 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., storeName=5858b23891076d8cb7718291e3b42567/info, priority=13, startTime=1732001815484; duration=0sec
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.8 K, sizeToCheck=16.0 K
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 because midkey is the same as first or last row
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.8 K, sizeToCheck=16.0 K
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 because midkey is the same as first or last row
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.8 K, sizeToCheck=16.0 K
2024-11-19T07:36:55,549 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:55,550 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 because midkey is the same as first or last row
2024-11-19T07:36:55,550 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:36:55,550 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5858b23891076d8cb7718291e3b42567:info
2024-11-19T07:36:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T07:36:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T07:36:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 because the exception is null or not the one we care about
2024-11-19T07:36:55,915 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/9c7214f34a9f4956baa80ef6d5c4f4ca
2024-11-19T07:36:55,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/9c7214f34a9f4956baa80ef6d5c4f4ca as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca
2024-11-19T07:36:55,927 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca, entries=13, sequenceid=60, filesize=18.5 K
2024-11-19T07:36:55,928 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=16.81 KB/17216 for 5858b23891076d8cb7718291e3b42567 in 442ms, sequenceid=60, compaction requested=false
2024-11-19T07:36:55,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:36:55,928 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-19T07:36:55,928 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:36:55,928 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 because midkey is the same as first or last row
2024-11-19T07:36:56,152 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:56,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:57,152 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:57,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:58,153 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:58,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:59,154 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:36:59,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:00,154 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:00,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:01,155 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:01,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:02,155 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:02,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:03,156 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:03,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:04,157 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:04,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:05,157 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:05,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:05,609 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5858b23891076d8cb7718291e3b42567 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-19T07:37:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/e4acd3536c4744e6abc8a059f75a71f6 is 1080, key is row0048/info:/1732001815488/Put/seqid=0
2024-11-19T07:37:05,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741842_1018 (size=23299)
2024-11-19T07:37:05,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741842_1018 (size=23299)
2024-11-19T07:37:05,619 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/e4acd3536c4744e6abc8a059f75a71f6
2024-11-19T07:37:05,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/e4acd3536c4744e6abc8a059f75a71f6 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6
2024-11-19T07:37:05,629 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6, entries=17, sequenceid=81, filesize=22.8 K
2024-11-19T07:37:05,630 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=0 B/0 for 5858b23891076d8cb7718291e3b42567 in 21ms, sequenceid=81, compaction requested=true
2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.1 K, sizeToCheck=16.0 K
2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 because midkey is the same as first or last row
2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5858b23891076d8cb7718291e3b42567:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:05,630 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:05,631 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 84033 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:05,631 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1541): 5858b23891076d8cb7718291e3b42567/info is initiating minor compaction (all files)
2024-11-19T07:37:05,631 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5858b23891076d8cb7718291e3b42567/info in TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:37:05,632 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp, totalSize=82.1 K
2024-11-19T07:37:05,632 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ffea9526ef4345bb38edf524b0f838, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732001813385
2024-11-19T07:37:05,632 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7214f34a9f4956baa80ef6d5c4f4ca, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732001815454
2024-11-19T07:37:05,633 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.Compactor(225): Compacting e4acd3536c4744e6abc8a059f75a71f6, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732001815488
2024-11-19T07:37:05,643 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5858b23891076d8cb7718291e3b42567#info#compaction#63 average throughput is 21.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:05,644 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/be2905e9e98b4ca48a40e266893f5525 is 1080, key is row0001/info:/1732001813385/Put/seqid=0
2024-11-19T07:37:05,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741843_1019 (size=74301)
2024-11-19T07:37:05,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741843_1019 (size=74301)
2024-11-19T07:37:05,652 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/.tmp/info/be2905e9e98b4ca48a40e266893f5525 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525
2024-11-19T07:37:05,657 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5858b23891076d8cb7718291e3b42567/info of 5858b23891076d8cb7718291e3b42567 into be2905e9e98b4ca48a40e266893f5525(size=72.6 K), total size for store is 72.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:05,657 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5858b23891076d8cb7718291e3b42567:

2024-11-19T07:37:05,658 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., storeName=5858b23891076d8cb7718291e3b42567/info, priority=13, startTime=1732001825630; duration=0sec
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-19T07:37:05,658 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-19T07:37:05,659 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:05,659 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:05,659 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5858b23891076d8cb7718291e3b42567:info
2024-11-19T07:37:05,660 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37997 {}] assignment.AssignmentManager(1355): Split request from db0afee3eab9,36609,1732001801933, parent={ENCODED => 5858b23891076d8cb7718291e3b42567, NAME => 'TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-19T07:37:05,665 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37997 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:05,668 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37997 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5858b23891076d8cb7718291e3b42567, daughterA=45979653a3c847bd948c26872806501f, daughterB=d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:05,669 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5858b23891076d8cb7718291e3b42567, daughterA=45979653a3c847bd948c26872806501f, daughterB=d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:05,669 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5858b23891076d8cb7718291e3b42567, daughterA=45979653a3c847bd948c26872806501f, daughterB=d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:05,669 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5858b23891076d8cb7718291e3b42567, daughterA=45979653a3c847bd948c26872806501f, daughterB=d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:05,676 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, UNASSIGN}]
2024-11-19T07:37:05,678 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, UNASSIGN
2024-11-19T07:37:05,679 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5858b23891076d8cb7718291e3b42567, regionState=CLOSING, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:05,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, UNASSIGN because future has completed
2024-11-19T07:37:05,682 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-19T07:37:05,682 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933}]
2024-11-19T07:37:05,839 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:05,839 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-19T07:37:05,840 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 5858b23891076d8cb7718291e3b42567, disabling compactions & flushes
2024-11-19T07:37:05,840 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:37:05,840 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:37:05,840 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567. after waiting 0 ms
2024-11-19T07:37:05,840 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:37:05,841 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6] to archive
2024-11-19T07:37:05,842 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:37:05,843 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/8412e0dd69b14d7fb89ec26f06e11906
2024-11-19T07:37:05,844 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/057fbb019e994407b37c9b08dd76d606
2024-11-19T07:37:05,845 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/50ffea9526ef4345bb38edf524b0f838
2024-11-19T07:37:05,846 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/627b4d117a5f41cfb8986e773afb2c69
2024-11-19T07:37:05,848 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/9c7214f34a9f4956baa80ef6d5c4f4ca
2024-11-19T07:37:05,849 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/e4acd3536c4744e6abc8a059f75a71f6
2024-11-19T07:37:05,856 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=1
2024-11-19T07:37:05,857 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.
2024-11-19T07:37:05,857 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 5858b23891076d8cb7718291e3b42567:
Waiting for close lock at 1732001825840Running coprocessor pre-close hooks at 1732001825840Disabling compacts and flushes for region at 1732001825840Disabling writes for close at 1732001825840Writing region close event to WAL at 1732001825852 (+12 ms)Running coprocessor post-close hooks at 1732001825857 (+5 ms)Closed at 1732001825857
2024-11-19T07:37:05,860 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:05,861 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5858b23891076d8cb7718291e3b42567, regionState=CLOSED
2024-11-19T07:37:05,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933 because future has completed
2024-11-19T07:37:05,868 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-19T07:37:05,868 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 5858b23891076d8cb7718291e3b42567, server=db0afee3eab9,36609,1732001801933 in 183 msec
2024-11-19T07:37:05,871 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-19T07:37:05,871 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5858b23891076d8cb7718291e3b42567, UNASSIGN in 192 msec
2024-11-19T07:37:05,882 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:05,884 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 1 storefiles, region=5858b23891076d8cb7718291e3b42567, threads=1
2024-11-19T07:37:05,886 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525 for region: 5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:05,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741844_1020 (size=27)
2024-11-19T07:37:05,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741844_1020 (size=27)
2024-11-19T07:37:05,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741845_1021 (size=27)
2024-11-19T07:37:05,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741845_1021 (size=27)
2024-11-19T07:37:05,906 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525 for region: 5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:05,908 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 5858b23891076d8cb7718291e3b42567 Daughter A: [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567] storefiles, Daughter B: [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567] storefiles.
2024-11-19T07:37:05,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741846_1022 (size=71)
2024-11-19T07:37:05,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741846_1022 (size=71)
2024-11-19T07:37:05,918 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:05,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741847_1023 (size=71)
2024-11-19T07:37:05,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741847_1023 (size=71)
2024-11-19T07:37:05,934 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:05,944 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-11-19T07:37:05,946 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-11-19T07:37:05,950 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732001825949"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732001825949"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732001825949"}]},"ts":"1732001825949"}
2024-11-19T07:37:05,950 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732001825949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001825949"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732001825949"}]},"ts":"1732001825949"}
2024-11-19T07:37:05,950 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732001825949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732001825949"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732001825949"}]},"ts":"1732001825949"}
2024-11-19T07:37:05,968 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=45979653a3c847bd948c26872806501f, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b0d555b374134379e8a6f81b8ef210, ASSIGN}]
2024-11-19T07:37:05,970 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b0d555b374134379e8a6f81b8ef210, ASSIGN
2024-11-19T07:37:05,970 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=45979653a3c847bd948c26872806501f, ASSIGN
2024-11-19T07:37:05,971 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b0d555b374134379e8a6f81b8ef210, ASSIGN; state=SPLITTING_NEW, location=db0afee3eab9,36609,1732001801933; forceNewPlan=false, retain=false
2024-11-19T07:37:05,971 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=45979653a3c847bd948c26872806501f, ASSIGN; state=SPLITTING_NEW, location=db0afee3eab9,36609,1732001801933; forceNewPlan=false, retain=false
2024-11-19T07:37:06,122 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=d6b0d555b374134379e8a6f81b8ef210, regionState=OPENING, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:06,122 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=45979653a3c847bd948c26872806501f, regionState=OPENING, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:06,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b0d555b374134379e8a6f81b8ef210, ASSIGN because future has completed
2024-11-19T07:37:06,129 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6b0d555b374134379e8a6f81b8ef210, server=db0afee3eab9,36609,1732001801933}]
2024-11-19T07:37:06,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=45979653a3c847bd948c26872806501f, ASSIGN because future has completed
2024-11-19T07:37:06,132 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45979653a3c847bd948c26872806501f, server=db0afee3eab9,36609,1732001801933}]
2024-11-19T07:37:06,158 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:06,286 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:06,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => d6b0d555b374134379e8a6f81b8ef210, NAME => 'TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-19T07:37:06,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:06,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,287 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,288 INFO  [StoreOpener-d6b0d555b374134379e8a6f81b8ef210-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d6b0d555b374134379e8a6f81b8ef210 
2024-11-19T07:37:06,289 INFO  [StoreOpener-d6b0d555b374134379e8a6f81b8ef210-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6b0d555b374134379e8a6f81b8ef210 columnFamilyName info
2024-11-19T07:37:06,289 DEBUG [StoreOpener-d6b0d555b374134379e8a6f81b8ef210-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:06,297 DEBUG [StoreOpener-d6b0d555b374134379e8a6f81b8ef210-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-top
2024-11-19T07:37:06,298 INFO  [StoreOpener-d6b0d555b374134379e8a6f81b8ef210-1 {}] regionserver.HStore(327): Store=d6b0d555b374134379e8a6f81b8ef210/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:37:06,298 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,299 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,300 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,300 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,300 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,301 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,302 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened d6b0d555b374134379e8a6f81b8ef210; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881638, jitterRate=0.12106116116046906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:37:06,302 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:06,303 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for d6b0d555b374134379e8a6f81b8ef210:
Running coprocessor pre-open hook at 1732001826287Writing region info on filesystem at 1732001826287Initializing all the Stores at 1732001826288 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001826288Cleaning up temporary data from old regions at 1732001826300 (+12 ms)Running coprocessor post-open hooks at 1732001826302 (+2 ms)Region opened successfully at 1732001826303 (+1 ms)
2024-11-19T07:37:06,303 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., pid=12, masterSystemTime=1732001826283
2024-11-19T07:37:06,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:06,304 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:06,304 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-19T07:37:06,304 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:06,304 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:06,305 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:06,305 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-top] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=72.6 K
2024-11-19T07:37:06,305 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732001813385
2024-11-19T07:37:06,306 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:06,306 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:06,306 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:06,306 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 45979653a3c847bd948c26872806501f, NAME => 'TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-19T07:37:06,306 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,307 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:06,307 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,307 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,307 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=d6b0d555b374134379e8a6f81b8ef210, regionState=OPEN, openSeqNum=86, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:06,308 INFO  [StoreOpener-45979653a3c847bd948c26872806501f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 45979653a3c847bd948c26872806501f 
2024-11-19T07:37:06,309 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-19T07:37:06,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-19T07:37:06,309 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-19T07:37:06,309 INFO  [StoreOpener-45979653a3c847bd948c26872806501f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45979653a3c847bd948c26872806501f columnFamilyName info
2024-11-19T07:37:06,309 DEBUG [StoreOpener-45979653a3c847bd948c26872806501f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:06,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6b0d555b374134379e8a6f81b8ef210, server=db0afee3eab9,36609,1732001801933 because future has completed
2024-11-19T07:37:06,313 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-19T07:37:06,313 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure d6b0d555b374134379e8a6f81b8ef210, server=db0afee3eab9,36609,1732001801933 in 182 msec
2024-11-19T07:37:06,315 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b0d555b374134379e8a6f81b8ef210, ASSIGN in 345 msec
2024-11-19T07:37:06,323 DEBUG [StoreOpener-45979653a3c847bd948c26872806501f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-bottom
2024-11-19T07:37:06,323 INFO  [StoreOpener-45979653a3c847bd948c26872806501f-1 {}] regionserver.HStore(327): Store=45979653a3c847bd948c26872806501f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:37:06,323 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,325 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,326 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,328 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,329 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 45979653a3c847bd948c26872806501f; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871540, jitterRate=0.10822159051895142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-19T07:37:06,329 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:06,329 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 45979653a3c847bd948c26872806501f:
Running coprocessor pre-open hook at 1732001826307Writing region info on filesystem at 1732001826307Initializing all the Stores at 1732001826307Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001826307Cleaning up temporary data from old regions at 1732001826326 (+19 ms)Running coprocessor post-open hooks at 1732001826329 (+3 ms)Region opened successfully at 1732001826329
2024-11-19T07:37:06,330 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f., pid=13, masterSystemTime=1732001826283
2024-11-19T07:37:06,330 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 45979653a3c847bd948c26872806501f:info, priority=-2147483648, current under compaction store size is 2
2024-11-19T07:37:06,330 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-19T07:37:06,330 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:06,331 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:06,331 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1541): 45979653a3c847bd948c26872806501f/info is initiating minor compaction (all files)
2024-11-19T07:37:06,331 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45979653a3c847bd948c26872806501f/info in TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:06,331 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-bottom] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/.tmp, totalSize=72.6 K
2024-11-19T07:37:06,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/81fff1b942ed406c885f805f13c1aeb9 is 193, key is TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210./info:regioninfo/1732001826307/Put/seqid=0
2024-11-19T07:37:06,333 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#65 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:06,333 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] compactions.Compactor(225): Compacting be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732001813385
2024-11-19T07:37:06,333 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/1cba198dad964c4fa46471f9792e9ef9 is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:06,334 DEBUG [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:06,334 INFO  [RS_OPEN_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:06,335 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=45979653a3c847bd948c26872806501f, regionState=OPEN, openSeqNum=86, regionLocation=db0afee3eab9,36609,1732001801933
2024-11-19T07:37:06,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45979653a3c847bd948c26872806501f, server=db0afee3eab9,36609,1732001801933 because future has completed
2024-11-19T07:37:06,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:06,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741848_1024 (size=9882)
2024-11-19T07:37:06,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741848_1024 (size=9882)
2024-11-19T07:37:06,353 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-19T07:37:06,353 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 45979653a3c847bd948c26872806501f, server=db0afee3eab9,36609,1732001801933 in 218 msec
2024-11-19T07:37:06,356 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-19T07:37:06,356 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=45979653a3c847bd948c26872806501f, ASSIGN in 385 msec
2024-11-19T07:37:06,358 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=5858b23891076d8cb7718291e3b42567, daughterA=45979653a3c847bd948c26872806501f, daughterB=d6b0d555b374134379e8a6f81b8ef210 in 691 msec
2024-11-19T07:37:06,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741849_1025 (size=8260)
2024-11-19T07:37:06,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741849_1025 (size=8260)
2024-11-19T07:37:06,364 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45979653a3c847bd948c26872806501f#info#compaction#66 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:06,364 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/.tmp/info/6d2f142cc9e848aa90d124d4c57a1221 is 1080, key is row0001/info:/1732001813385/Put/seqid=0
2024-11-19T07:37:06,365 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/1cba198dad964c4fa46471f9792e9ef9 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1cba198dad964c4fa46471f9792e9ef9
2024-11-19T07:37:06,372 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 1cba198dad964c4fa46471f9792e9ef9(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:06,372 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:06,372 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=15, startTime=1732001826304; duration=0sec
2024-11-19T07:37:06,372 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:06,372 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:06,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741850_1026 (size=70862)
2024-11-19T07:37:06,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741850_1026 (size=70862)
2024-11-19T07:37:06,385 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/.tmp/info/6d2f142cc9e848aa90d124d4c57a1221 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/6d2f142cc9e848aa90d124d4c57a1221
2024-11-19T07:37:06,392 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 45979653a3c847bd948c26872806501f/info of 45979653a3c847bd948c26872806501f into 6d2f142cc9e848aa90d124d4c57a1221(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:06,392 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45979653a3c847bd948c26872806501f:

2024-11-19T07:37:06,392 INFO  [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f., storeName=45979653a3c847bd948c26872806501f/info, priority=15, startTime=1732001826330; duration=0sec
2024-11-19T07:37:06,392 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:06,392 DEBUG [RS:0;db0afee3eab9:36609-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45979653a3c847bd948c26872806501f:info
2024-11-19T07:37:06,751 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/81fff1b942ed406c885f805f13c1aeb9
2024-11-19T07:37:06,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/ns/8d3fbf33047d4460b41bef24ff473bb6 is 43, key is default/ns:d/1732001803222/Put/seqid=0
2024-11-19T07:37:06,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741851_1027 (size=5153)
2024-11-19T07:37:06,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741851_1027 (size=5153)
2024-11-19T07:37:06,778 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/ns/8d3fbf33047d4460b41bef24ff473bb6
2024-11-19T07:37:06,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/table/9fc42ddc55954c18a05c4151c8134f85 is 65, key is TestLogRolling-testLogRolling/table:state/1732001803648/Put/seqid=0
2024-11-19T07:37:06,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741852_1028 (size=5340)
2024-11-19T07:37:06,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741852_1028 (size=5340)
2024-11-19T07:37:06,806 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/table/9fc42ddc55954c18a05c4151c8134f85
2024-11-19T07:37:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/81fff1b942ed406c885f805f13c1aeb9 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/info/81fff1b942ed406c885f805f13c1aeb9
2024-11-19T07:37:06,817 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/info/81fff1b942ed406c885f805f13c1aeb9, entries=30, sequenceid=17, filesize=9.7 K
2024-11-19T07:37:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/ns/8d3fbf33047d4460b41bef24ff473bb6 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/ns/8d3fbf33047d4460b41bef24ff473bb6
2024-11-19T07:37:06,823 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/ns/8d3fbf33047d4460b41bef24ff473bb6, entries=2, sequenceid=17, filesize=5.0 K
2024-11-19T07:37:06,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/table/9fc42ddc55954c18a05c4151c8134f85 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/table/9fc42ddc55954c18a05c4151c8134f85
2024-11-19T07:37:06,835 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/table/9fc42ddc55954c18a05c4151c8134f85, entries=2, sequenceid=17, filesize=5.2 K
2024-11-19T07:37:06,836 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 527ms, sequenceid=17, compaction requested=false
2024-11-19T07:37:06,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T07:37:07,158 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:07,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36538 deadline: 1732001837610, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567. is not online on db0afee3eab9,36609,1732001801933
2024-11-19T07:37:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567. is not online on db0afee3eab9,36609,1732001801933
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T07:37:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567. is not online on db0afee3eab9,36609,1732001801933
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-19T07:37:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732001803279.5858b23891076d8cb7718291e3b42567., hostname=db0afee3eab9,36609,1732001801933, seqNum=2 from cache
2024-11-19T07:37:08,159 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:08,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:09,160 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:09,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:10,160 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:10,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:10,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,859 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,859 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,862 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,862 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:10,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:11,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:11,405 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-19T07:37:11,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,408 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,408 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,436 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,436 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,444 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,444 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,449 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-19T07:37:11,754 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-19T07:37:12,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:12,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:13,162 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:13,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:14,163 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:14,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:15,163 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:15,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:16,164 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:16,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:17,165 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:17,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., hostname=db0afee3eab9,36609,1732001801933, seqNum=86]
2024-11-19T07:37:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:17,715 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:37:17,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/0640bb125c2341d3b781078391e502d5 is 1080, key is row0065/info:/1732001837701/Put/seqid=0
2024-11-19T07:37:17,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741853_1029 (size=12509)
2024-11-19T07:37:17,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741853_1029 (size=12509)
2024-11-19T07:37:17,737 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/0640bb125c2341d3b781078391e502d5
2024-11-19T07:37:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/0640bb125c2341d3b781078391e502d5 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5
2024-11-19T07:37:17,750 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5, entries=7, sequenceid=96, filesize=12.2 K
2024-11-19T07:37:17,751 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for d6b0d555b374134379e8a6f81b8ef210 in 36ms, sequenceid=96, compaction requested=false
2024-11-19T07:37:17,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:17,753 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-19T07:37:17,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/de7c94e83832450c8f8553bfc9d13ea1 is 1080, key is row0072/info:/1732001837716/Put/seqid=0
2024-11-19T07:37:17,762 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741854_1030 (size=25453)
2024-11-19T07:37:17,762 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741854_1030 (size=25453)
2024-11-19T07:37:18,164 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/de7c94e83832450c8f8553bfc9d13ea1
2024-11-19T07:37:18,166 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:18,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/de7c94e83832450c8f8553bfc9d13ea1 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1
2024-11-19T07:37:18,182 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1, entries=19, sequenceid=118, filesize=24.9 K
2024-11-19T07:37:18,184 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for d6b0d555b374134379e8a6f81b8ef210 in 430ms, sequenceid=118, compaction requested=true
2024-11-19T07:37:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:18,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:18,184 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:18,185 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 46222 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:18,185 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:18,185 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:18,186 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1cba198dad964c4fa46471f9792e9ef9, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=45.1 K
2024-11-19T07:37:18,186 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cba198dad964c4fa46471f9792e9ef9, keycount=3, bloomtype=ROW, size=8.1 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732001815537
2024-11-19T07:37:18,187 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0640bb125c2341d3b781078391e502d5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732001837701
2024-11-19T07:37:18,187 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting de7c94e83832450c8f8553bfc9d13ea1, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732001837716
2024-11-19T07:37:18,198 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#71 average throughput is 29.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:18,198 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/675b003f81a4427794d98bd87165cc6e is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:18,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741855_1031 (size=36426)
2024-11-19T07:37:18,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741855_1031 (size=36426)
2024-11-19T07:37:18,208 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/675b003f81a4427794d98bd87165cc6e as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/675b003f81a4427794d98bd87165cc6e
2024-11-19T07:37:18,215 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 675b003f81a4427794d98bd87165cc6e(size=35.6 K), total size for store is 35.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:18,215 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:18,215 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001838184; duration=0sec
2024-11-19T07:37:18,215 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:18,215 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:18,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:19,167 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:19,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:19,770 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:37:19,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/1c99ced9ca804d79ba42b70d780d7fcf is 1080, key is row0091/info:/1732001837754/Put/seqid=0
2024-11-19T07:37:19,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741856_1032 (size=12510)
2024-11-19T07:37:19,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741856_1032 (size=12510)
2024-11-19T07:37:19,784 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/1c99ced9ca804d79ba42b70d780d7fcf
2024-11-19T07:37:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/1c99ced9ca804d79ba42b70d780d7fcf as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf
2024-11-19T07:37:19,796 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf, entries=7, sequenceid=129, filesize=12.2 K
2024-11-19T07:37:19,797 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for d6b0d555b374134379e8a6f81b8ef210 in 26ms, sequenceid=129, compaction requested=false
2024-11-19T07:37:19,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:19,797 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T07:37:19,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/b6aab149b2564b659cdc147c546e6c6e is 1080, key is row0098/info:/1732001839772/Put/seqid=0
2024-11-19T07:37:19,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741857_1033 (size=16828)
2024-11-19T07:37:19,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741857_1033 (size=16828)
2024-11-19T07:37:19,806 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/b6aab149b2564b659cdc147c546e6c6e
2024-11-19T07:37:19,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/b6aab149b2564b659cdc147c546e6c6e as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e
2024-11-19T07:37:19,817 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e, entries=11, sequenceid=143, filesize=16.4 K
2024-11-19T07:37:19,818 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for d6b0d555b374134379e8a6f81b8ef210 in 21ms, sequenceid=143, compaction requested=true
2024-11-19T07:37:19,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:19,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:19,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:19,818 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:19,819 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T07:37:19,820 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 65764 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:19,820 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:19,820 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:19,820 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/675b003f81a4427794d98bd87165cc6e, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=64.2 K
2024-11-19T07:37:19,820 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675b003f81a4427794d98bd87165cc6e, keycount=29, bloomtype=ROW, size=35.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732001815537
2024-11-19T07:37:19,821 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c99ced9ca804d79ba42b70d780d7fcf, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732001837754
2024-11-19T07:37:19,821 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6aab149b2564b659cdc147c546e6c6e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732001839772
2024-11-19T07:37:19,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/20b77be955e44c91877fce77b4b9daa3 is 1080, key is row0109/info:/1732001839798/Put/seqid=0
2024-11-19T07:37:19,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741858_1034 (size=16828)
2024-11-19T07:37:19,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741858_1034 (size=16828)
2024-11-19T07:37:19,831 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/20b77be955e44c91877fce77b4b9daa3
2024-11-19T07:37:19,837 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#75 average throughput is 24.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:19,837 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/be51359bfe984b02b85b7aa5fc488aad is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:19,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/20b77be955e44c91877fce77b4b9daa3 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3
2024-11-19T07:37:19,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741859_1035 (size=55934)
2024-11-19T07:37:19,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741859_1035 (size=55934)
2024-11-19T07:37:19,843 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3, entries=11, sequenceid=157, filesize=16.4 K
2024-11-19T07:37:19,844 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for d6b0d555b374134379e8a6f81b8ef210 in 25ms, sequenceid=157, compaction requested=false
2024-11-19T07:37:19,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:19,848 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/be51359bfe984b02b85b7aa5fc488aad as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be51359bfe984b02b85b7aa5fc488aad
2024-11-19T07:37:19,853 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into be51359bfe984b02b85b7aa5fc488aad(size=54.6 K), total size for store is 71.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:19,853 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:19,853 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001839818; duration=0sec
2024-11-19T07:37:19,853 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:19,853 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:20,169 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:20,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:21,170 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:21,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:21,845 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-19T07:37:21,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/73243dc926db4862bdd38548a9c908ff is 1080, key is row0120/info:/1732001839820/Put/seqid=0
2024-11-19T07:37:21,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741860_1036 (size=15750)
2024-11-19T07:37:21,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741860_1036 (size=15750)
2024-11-19T07:37:21,860 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/73243dc926db4862bdd38548a9c908ff
2024-11-19T07:37:21,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/73243dc926db4862bdd38548a9c908ff as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff
2024-11-19T07:37:21,871 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff, entries=10, sequenceid=171, filesize=15.4 K
2024-11-19T07:37:21,872 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=10.51 KB/10760 for d6b0d555b374134379e8a6f81b8ef210 in 27ms, sequenceid=171, compaction requested=true
2024-11-19T07:37:21,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:21,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:21,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:21,872 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:21,873 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T07:37:21,873 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 88512 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:21,874 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:21,874 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:21,874 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be51359bfe984b02b85b7aa5fc488aad, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=86.4 K
2024-11-19T07:37:21,874 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting be51359bfe984b02b85b7aa5fc488aad, keycount=47, bloomtype=ROW, size=54.6 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732001815537
2024-11-19T07:37:21,875 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b77be955e44c91877fce77b4b9daa3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732001839798
2024-11-19T07:37:21,875 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73243dc926db4862bdd38548a9c908ff, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732001839820
2024-11-19T07:37:21,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/26efb6dbb360474682ebb6ee16495d82 is 1080, key is row0130/info:/1732001841848/Put/seqid=0
2024-11-19T07:37:21,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741861_1037 (size=16828)
2024-11-19T07:37:21,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741861_1037 (size=16828)
2024-11-19T07:37:21,884 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/26efb6dbb360474682ebb6ee16495d82
2024-11-19T07:37:21,887 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#78 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:21,888 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/dbdbdbe46cbe4e55bca1494a7badbaf0 is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:21,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/26efb6dbb360474682ebb6ee16495d82 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82
2024-11-19T07:37:21,892 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741862_1038 (size=78811)
2024-11-19T07:37:21,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741862_1038 (size=78811)
2024-11-19T07:37:21,896 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82, entries=11, sequenceid=185, filesize=16.4 K
2024-11-19T07:37:21,897 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for d6b0d555b374134379e8a6f81b8ef210 in 24ms, sequenceid=185, compaction requested=false
2024-11-19T07:37:21,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:21,898 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/dbdbdbe46cbe4e55bca1494a7badbaf0 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/dbdbdbe46cbe4e55bca1494a7badbaf0
2024-11-19T07:37:21,898 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-19T07:37:21,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/00318f2c243f484084f36e13d3d09177 is 1080, key is row0141/info:/1732001841874/Put/seqid=0
2024-11-19T07:37:21,905 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into dbdbdbe46cbe4e55bca1494a7badbaf0(size=77.0 K), total size for store is 93.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:21,905 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:21,905 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001841872; duration=0sec
2024-11-19T07:37:21,905 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:21,905 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:21,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741863_1039 (size=17906)
2024-11-19T07:37:21,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741863_1039 (size=17906)
2024-11-19T07:37:21,910 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/00318f2c243f484084f36e13d3d09177
2024-11-19T07:37:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/00318f2c243f484084f36e13d3d09177 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177
2024-11-19T07:37:21,920 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177, entries=12, sequenceid=200, filesize=17.5 K
2024-11-19T07:37:21,921 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=8.41 KB/8608 for d6b0d555b374134379e8a6f81b8ef210 in 23ms, sequenceid=200, compaction requested=true
2024-11-19T07:37:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:21,921 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:21,922 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 113545 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:21,922 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:21,922 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:21,923 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/dbdbdbe46cbe4e55bca1494a7badbaf0, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=110.9 K
2024-11-19T07:37:21,923 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbdbdbe46cbe4e55bca1494a7badbaf0, keycount=68, bloomtype=ROW, size=77.0 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732001815537
2024-11-19T07:37:21,923 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26efb6dbb360474682ebb6ee16495d82, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732001841848
2024-11-19T07:37:21,923 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00318f2c243f484084f36e13d3d09177, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732001841874
2024-11-19T07:37:21,933 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#80 average throughput is 46.69 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:21,933 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/7c8dbcb421a14369a6fe0fd792c2c04b is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:21,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741864_1040 (size=103711)
2024-11-19T07:37:21,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741864_1040 (size=103711)
2024-11-19T07:37:21,941 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/7c8dbcb421a14369a6fe0fd792c2c04b as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/7c8dbcb421a14369a6fe0fd792c2c04b
2024-11-19T07:37:21,947 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 7c8dbcb421a14369a6fe0fd792c2c04b(size=101.3 K), total size for store is 101.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:21,947 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:21,947 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001841921; duration=0sec
2024-11-19T07:37:21,947 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:21,947 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:22,170 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:22,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:23,171 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:23,238 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-19T07:37:23,239 INFO  [master/db0afee3eab9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-19T07:37:23,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:23,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-19T07:37:23,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/d79595d49f5d41879c96b833c9d7e4c2 is 1080, key is row0153/info:/1732001841899/Put/seqid=0
2024-11-19T07:37:23,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741865_1041 (size=14672)
2024-11-19T07:37:23,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741865_1041 (size=14672)
2024-11-19T07:37:23,943 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/d79595d49f5d41879c96b833c9d7e4c2
2024-11-19T07:37:23,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/d79595d49f5d41879c96b833c9d7e4c2 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2
2024-11-19T07:37:23,956 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2, entries=9, sequenceid=214, filesize=14.3 K
2024-11-19T07:37:23,957 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=12.61 KB/12912 for d6b0d555b374134379e8a6f81b8ef210 in 25ms, sequenceid=214, compaction requested=false
2024-11-19T07:37:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:23,958 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-19T07:37:23,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/99fb7e67d46640a48a8c586e0fbc9e0c is 1080, key is row0162/info:/1732001843934/Put/seqid=0
2024-11-19T07:37:23,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741866_1042 (size=19000)
2024-11-19T07:37:23,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741866_1042 (size=19000)
2024-11-19T07:37:23,968 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/99fb7e67d46640a48a8c586e0fbc9e0c
2024-11-19T07:37:23,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/99fb7e67d46640a48a8c586e0fbc9e0c as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c
2024-11-19T07:37:23,978 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c, entries=13, sequenceid=230, filesize=18.6 K
2024-11-19T07:37:23,979 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for d6b0d555b374134379e8a6f81b8ef210 in 21ms, sequenceid=230, compaction requested=true
2024-11-19T07:37:23,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:23,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:23,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:23,980 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:23,980 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T07:37:23,981 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 137383 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:23,981 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:23,981 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:23,981 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/7c8dbcb421a14369a6fe0fd792c2c04b, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=134.2 K
2024-11-19T07:37:23,982 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8dbcb421a14369a6fe0fd792c2c04b, keycount=91, bloomtype=ROW, size=101.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732001815537
2024-11-19T07:37:23,982 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79595d49f5d41879c96b833c9d7e4c2, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732001841899
2024-11-19T07:37:23,982 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99fb7e67d46640a48a8c586e0fbc9e0c, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732001843934
2024-11-19T07:37:23,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/5060b84c115c4f6ca45e92460474063b is 1080, key is row0175/info:/1732001843959/Put/seqid=0
2024-11-19T07:37:23,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741867_1043 (size=16828)
2024-11-19T07:37:23,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741867_1043 (size=16828)
2024-11-19T07:37:23,991 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/5060b84c115c4f6ca45e92460474063b
2024-11-19T07:37:24,007 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#84 average throughput is 28.99 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/5060b84c115c4f6ca45e92460474063b as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b
2024-11-19T07:37:24,008 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/c3ce67b1362442099688941e823bb0b3 is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:24,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741868_1044 (size=127665)
2024-11-19T07:37:24,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741868_1044 (size=127665)
2024-11-19T07:37:24,015 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b, entries=11, sequenceid=244, filesize=16.4 K
2024-11-19T07:37:24,016 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=7.36 KB/7532 for d6b0d555b374134379e8a6f81b8ef210 in 36ms, sequenceid=244, compaction requested=false
2024-11-19T07:37:24,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:24,017 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/c3ce67b1362442099688941e823bb0b3 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/c3ce67b1362442099688941e823bb0b3
2024-11-19T07:37:24,024 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into c3ce67b1362442099688941e823bb0b3(size=124.7 K), total size for store is 141.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:24,024 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:24,024 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001843980; duration=0sec
2024-11-19T07:37:24,024 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:24,024 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:24,173 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:24,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:25,174 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:25,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:26,010 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-19T07:37:26,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/75865ae56f354faabece6aa5ce4b2981 is 1080, key is row0186/info:/1732001843981/Put/seqid=0
2024-11-19T07:37:26,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741869_1045 (size=13594)
2024-11-19T07:37:26,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741869_1045 (size=13594)
2024-11-19T07:37:26,024 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/75865ae56f354faabece6aa5ce4b2981
2024-11-19T07:37:26,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/75865ae56f354faabece6aa5ce4b2981 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981
2024-11-19T07:37:26,035 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981, entries=8, sequenceid=256, filesize=13.3 K
2024-11-19T07:37:26,036 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for d6b0d555b374134379e8a6f81b8ef210 in 27ms, sequenceid=256, compaction requested=true
2024-11-19T07:37:26,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:26,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:26,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:26,036 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:26,037 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-19T07:37:26,038 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 158087 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:26,038 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:26,038 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:26,038 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/c3ce67b1362442099688941e823bb0b3, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=154.4 K
2024-11-19T07:37:26,039 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3ce67b1362442099688941e823bb0b3, keycount=113, bloomtype=ROW, size=124.7 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732001815537
2024-11-19T07:37:26,039 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5060b84c115c4f6ca45e92460474063b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732001843959
2024-11-19T07:37:26,039 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75865ae56f354faabece6aa5ce4b2981, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732001843981
2024-11-19T07:37:26,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/681983dd3ec446caaebf7fe205837805 is 1080, key is row0194/info:/1732001846011/Put/seqid=0
2024-11-19T07:37:26,053 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#87 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:26,053 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/0483b689a35942078bb209ddedbe828c is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:26,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741870_1046 (size=17918)
2024-11-19T07:37:26,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741870_1046 (size=17918)
2024-11-19T07:37:26,059 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/681983dd3ec446caaebf7fe205837805
2024-11-19T07:37:26,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741871_1047 (size=148322)
2024-11-19T07:37:26,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741871_1047 (size=148322)
2024-11-19T07:37:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/681983dd3ec446caaebf7fe205837805 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805
2024-11-19T07:37:26,067 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/0483b689a35942078bb209ddedbe828c as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0483b689a35942078bb209ddedbe828c
2024-11-19T07:37:26,071 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805, entries=12, sequenceid=271, filesize=17.5 K
2024-11-19T07:37:26,072 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=17.86 KB/18292 for d6b0d555b374134379e8a6f81b8ef210 in 35ms, sequenceid=271, compaction requested=false
2024-11-19T07:37:26,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:26,074 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 0483b689a35942078bb209ddedbe828c(size=144.8 K), total size for store is 162.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:26,074 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:26,074 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001846036; duration=0sec
2024-11-19T07:37:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:26,074 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-19T07:37:26,074 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:26,074 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:26,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/19b83451823942ce983f96ab9a8f6806 is 1080, key is row0206/info:/1732001846038/Put/seqid=0
2024-11-19T07:37:26,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741872_1048 (size=25491)
2024-11-19T07:37:26,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741872_1048 (size=25491)
2024-11-19T07:37:26,084 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/19b83451823942ce983f96ab9a8f6806
2024-11-19T07:37:26,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/19b83451823942ce983f96ab9a8f6806 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806
2024-11-19T07:37:26,094 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806, entries=19, sequenceid=294, filesize=24.9 K
2024-11-19T07:37:26,095 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=0 B/0 for d6b0d555b374134379e8a6f81b8ef210 in 20ms, sequenceid=294, compaction requested=true
2024-11-19T07:37:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:26,095 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:26,096 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 191731 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:26,096 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:26,096 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:26,096 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0483b689a35942078bb209ddedbe828c, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=187.2 K
2024-11-19T07:37:26,097 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0483b689a35942078bb209ddedbe828c, keycount=132, bloomtype=ROW, size=144.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732001815537
2024-11-19T07:37:26,097 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681983dd3ec446caaebf7fe205837805, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732001846011
2024-11-19T07:37:26,097 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b83451823942ce983f96ab9a8f6806, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732001846038
2024-11-19T07:37:26,109 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#89 average throughput is 55.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:26,109 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/79bc49229b534a28a4047d5b1be10e8c is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:26,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741873_1049 (size=181881)
2024-11-19T07:37:26,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741873_1049 (size=181881)
2024-11-19T07:37:26,119 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/79bc49229b534a28a4047d5b1be10e8c as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/79bc49229b534a28a4047d5b1be10e8c
2024-11-19T07:37:26,125 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 79bc49229b534a28a4047d5b1be10e8c(size=177.6 K), total size for store is 177.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:26,125 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:26,125 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001846095; duration=0sec
2024-11-19T07:37:26,125 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:26,125 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:26,175 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:26,355 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:27,175 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:27,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:28,097 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-19T07:37:28,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/4dcec10743e14263ad15613223ee726a is 1080, key is row0225/info:/1732001848077/Put/seqid=0
2024-11-19T07:37:28,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741874_1050 (size=12523)
2024-11-19T07:37:28,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741874_1050 (size=12523)
2024-11-19T07:37:28,108 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/4dcec10743e14263ad15613223ee726a
2024-11-19T07:37:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/4dcec10743e14263ad15613223ee726a as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a
2024-11-19T07:37:28,118 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a, entries=7, sequenceid=305, filesize=12.2 K
2024-11-19T07:37:28,119 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for d6b0d555b374134379e8a6f81b8ef210 in 22ms, sequenceid=305, compaction requested=false
2024-11-19T07:37:28,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36609 {}] regionserver.HRegion(8855): Flush requested on d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:28,121 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-19T07:37:28,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/6a97d652aa54483cafaf13424fd73bf0 is 1080, key is row0232/info:/1732001848098/Put/seqid=0
2024-11-19T07:37:28,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741875_1051 (size=16839)
2024-11-19T07:37:28,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741875_1051 (size=16839)
2024-11-19T07:37:28,153 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-11-19T07:37:28,176 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:28,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:28,532 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/6a97d652aa54483cafaf13424fd73bf0
2024-11-19T07:37:28,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/6a97d652aa54483cafaf13424fd73bf0 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0
2024-11-19T07:37:28,554 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0, entries=11, sequenceid=319, filesize=16.4 K
2024-11-19T07:37:28,556 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=14.71 KB/15064 for d6b0d555b374134379e8a6f81b8ef210 in 435ms, sequenceid=319, compaction requested=true
2024-11-19T07:37:28,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:28,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b0d555b374134379e8a6f81b8ef210:info, priority=-2147483648, current under compaction store size is 1
2024-11-19T07:37:28,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:28,556 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-19T07:37:28,557 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 211243 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-19T07:37:28,557 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1541): d6b0d555b374134379e8a6f81b8ef210/info is initiating minor compaction (all files)
2024-11-19T07:37:28,557 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b0d555b374134379e8a6f81b8ef210/info in TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:28,557 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/79bc49229b534a28a4047d5b1be10e8c, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0] into tmpdir=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp, totalSize=206.3 K
2024-11-19T07:37:28,558 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bc49229b534a28a4047d5b1be10e8c, keycount=163, bloomtype=ROW, size=177.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732001815537
2024-11-19T07:37:28,558 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dcec10743e14263ad15613223ee726a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732001848077
2024-11-19T07:37:28,559 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a97d652aa54483cafaf13424fd73bf0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732001848098
2024-11-19T07:37:28,570 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b0d555b374134379e8a6f81b8ef210#info#compaction#92 average throughput is 61.91 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-19T07:37:28,570 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/5a1c1d8ef0654e14badf8f2aa6644a2b is 1080, key is row0062/info:/1732001815537/Put/seqid=0
2024-11-19T07:37:28,572 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741876_1052 (size=201409)
2024-11-19T07:37:28,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741876_1052 (size=201409)
2024-11-19T07:37:28,577 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/5a1c1d8ef0654e14badf8f2aa6644a2b as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5a1c1d8ef0654e14badf8f2aa6644a2b
2024-11-19T07:37:28,583 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b0d555b374134379e8a6f81b8ef210/info of d6b0d555b374134379e8a6f81b8ef210 into 5a1c1d8ef0654e14badf8f2aa6644a2b(size=196.7 K), total size for store is 196.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-19T07:37:28,583 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:28,583 INFO  [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210., storeName=d6b0d555b374134379e8a6f81b8ef210/info, priority=13, startTime=1732001848556; duration=0sec
2024-11-19T07:37:28,583 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-19T07:37:28,583 DEBUG [RS:0;db0afee3eab9:36609-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b0d555b374134379e8a6f81b8ef210:info
2024-11-19T07:37:29,177 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:29,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:30,147 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-19T07:37:30,148 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C36609%2C1732001801933.1732001850147
2024-11-19T07:37:30,170 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,170 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,170 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,170 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,170 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,171 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001802661 with entries=314, filesize=308.87 KB; new WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001850147
2024-11-19T07:37:30,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741833_1009 (size=316287)
2024-11-19T07:37:30,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741833_1009 (size=316287)
2024-11-19T07:37:30,175 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33437:33437),(127.0.0.1/127.0.0.1:42025:42025)]
2024-11-19T07:37:30,177 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-19T07:37:30,178 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:30,181 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/90c1c7573d4949f5a86147e32bd6ff08 is 186, key is TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f./info:regioninfo/1732001826335/Put/seqid=0
2024-11-19T07:37:30,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741878_1054 (size=6153)
2024-11-19T07:37:30,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741878_1054 (size=6153)
2024-11-19T07:37:30,186 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/90c1c7573d4949f5a86147e32bd6ff08
2024-11-19T07:37:30,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/.tmp/info/90c1c7573d4949f5a86147e32bd6ff08 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/info/90c1c7573d4949f5a86147e32bd6ff08
2024-11-19T07:37:30,194 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/info/90c1c7573d4949f5a86147e32bd6ff08, entries=5, sequenceid=21, filesize=6.0 K
2024-11-19T07:37:30,195 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-11-19T07:37:30,196 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-19T07:37:30,196 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 45979653a3c847bd948c26872806501f:

2024-11-19T07:37:30,196 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing d6b0d555b374134379e8a6f81b8ef210 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-19T07:37:30,199 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/f3d0b8046bf54bfeaaef2435cbb60bb3 is 1080, key is row0243/info:/1732001848122/Put/seqid=0
2024-11-19T07:37:30,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741879_1055 (size=20092)
2024-11-19T07:37:30,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741879_1055 (size=20092)
2024-11-19T07:37:30,203 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/f3d0b8046bf54bfeaaef2435cbb60bb3
2024-11-19T07:37:30,208 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/.tmp/info/f3d0b8046bf54bfeaaef2435cbb60bb3 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/f3d0b8046bf54bfeaaef2435cbb60bb3
2024-11-19T07:37:30,212 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/f3d0b8046bf54bfeaaef2435cbb60bb3, entries=14, sequenceid=337, filesize=19.6 K
2024-11-19T07:37:30,213 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=0 B/0 for d6b0d555b374134379e8a6f81b8ef210 in 17ms, sequenceid=337, compaction requested=false
2024-11-19T07:37:30,213 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d6b0d555b374134379e8a6f81b8ef210:

2024-11-19T07:37:30,214 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C36609%2C1732001801933.1732001850213
2024-11-19T07:37:30,218 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,218 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,218 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,218 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,218 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,218 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001850147 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001850213
2024-11-19T07:37:30,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33437:33437),(127.0.0.1/127.0.0.1:42025:42025)]
2024-11-19T07:37:30,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001850147 is not closed yet, will try archiving it next time
2024-11-19T07:37:30,219 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001802661 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs/db0afee3eab9%2C36609%2C1732001801933.1732001802661
2024-11-19T07:37:30,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741877_1053 (size=731)
2024-11-19T07:37:30,220 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-19T07:37:30,221 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/WALs/db0afee3eab9,36609,1732001801933/db0afee3eab9%2C36609%2C1732001801933.1732001850147 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs/db0afee3eab9%2C36609%2C1732001801933.1732001850147
2024-11-19T07:37:30,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741877_1053 (size=731)
2024-11-19T07:37:30,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:37:30,320 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:37:30,320 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:30,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:30,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:30,321 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:37:30,321 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:37:30,321 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1779843503, stopped=false
2024-11-19T07:37:30,321 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,37997,1732001801772
2024-11-19T07:37:30,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:30,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:30,368 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:30,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:30,368 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:30,368 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:37:30,369 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:37:30,369 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:30,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:30,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:30,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:30,370 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,36609,1732001801933' *****
2024-11-19T07:37:30,370 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:37:30,371 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:37:30,371 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:37:30,371 INFO  [RS:0;db0afee3eab9:36609 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:37:30,371 INFO  [RS:0;db0afee3eab9:36609 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(3091): Received CLOSE for 45979653a3c847bd948c26872806501f
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(3091): Received CLOSE for d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,36609,1732001801933
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:37:30,372 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45979653a3c847bd948c26872806501f, disabling compactions & flushes
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:36609.
2024-11-19T07:37:30,372 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:30,372 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:30,372 DEBUG [RS:0;db0afee3eab9:36609 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:30,372 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f. after waiting 0 ms
2024-11-19T07:37:30,372 DEBUG [RS:0;db0afee3eab9:36609 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:30,372 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:30,372 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:37:30,373 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:37:30,373 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:37:30,373 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:37:30,373 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-19T07:37:30,373 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 45979653a3c847bd948c26872806501f=TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f., d6b0d555b374134379e8a6f81b8ef210=TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.}
2024-11-19T07:37:30,373 DEBUG [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 45979653a3c847bd948c26872806501f, d6b0d555b374134379e8a6f81b8ef210
2024-11-19T07:37:30,373 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:37:30,373 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-bottom] to archive
2024-11-19T07:37:30,373 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:37:30,373 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:37:30,374 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:37:30,374 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:37:30,375 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:37:30,377 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:30,378 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=db0afee3eab9:37997 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-19T07:37:30,378 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-19T07:37:30,379 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-19T07:37:30,380 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:37:30,380 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:37:30,380 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001850373Running coprocessor pre-close hooks at 1732001850373Disabling compacts and flushes for region at 1732001850373Disabling writes for close at 1732001850374 (+1 ms)Writing region close event to WAL at 1732001850376 (+2 ms)Running coprocessor post-close hooks at 1732001850380 (+4 ms)Closed at 1732001850380
2024-11-19T07:37:30,380 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:37:30,381 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/45979653a3c847bd948c26872806501f/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=85
2024-11-19T07:37:30,382 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:30,382 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45979653a3c847bd948c26872806501f:
Waiting for close lock at 1732001850372Running coprocessor pre-close hooks at 1732001850372Disabling compacts and flushes for region at 1732001850372Disabling writes for close at 1732001850372Writing region close event to WAL at 1732001850378 (+6 ms)Running coprocessor post-close hooks at 1732001850382 (+4 ms)Closed at 1732001850382
2024-11-19T07:37:30,382 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732001825665.45979653a3c847bd948c26872806501f.
2024-11-19T07:37:30,382 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6b0d555b374134379e8a6f81b8ef210, disabling compactions & flushes
2024-11-19T07:37:30,382 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:30,383 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:30,383 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210. after waiting 0 ms
2024-11-19T07:37:30,383 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:30,383 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567->hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/5858b23891076d8cb7718291e3b42567/info/be2905e9e98b4ca48a40e266893f5525-top, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1cba198dad964c4fa46471f9792e9ef9, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/675b003f81a4427794d98bd87165cc6e, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be51359bfe984b02b85b7aa5fc488aad, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/dbdbdbe46cbe4e55bca1494a7badbaf0, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/7c8dbcb421a14369a6fe0fd792c2c04b, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/c3ce67b1362442099688941e823bb0b3, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0483b689a35942078bb209ddedbe828c, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/79bc49229b534a28a4047d5b1be10e8c, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0] to archive
2024-11-19T07:37:30,384 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-19T07:37:30,386 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be2905e9e98b4ca48a40e266893f5525.5858b23891076d8cb7718291e3b42567
2024-11-19T07:37:30,387 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1cba198dad964c4fa46471f9792e9ef9 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1cba198dad964c4fa46471f9792e9ef9
2024-11-19T07:37:30,388 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0640bb125c2341d3b781078391e502d5
2024-11-19T07:37:30,389 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/675b003f81a4427794d98bd87165cc6e to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/675b003f81a4427794d98bd87165cc6e
2024-11-19T07:37:30,390 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/de7c94e83832450c8f8553bfc9d13ea1
2024-11-19T07:37:30,391 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/1c99ced9ca804d79ba42b70d780d7fcf
2024-11-19T07:37:30,392 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be51359bfe984b02b85b7aa5fc488aad to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/be51359bfe984b02b85b7aa5fc488aad
2024-11-19T07:37:30,393 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/b6aab149b2564b659cdc147c546e6c6e
2024-11-19T07:37:30,394 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/20b77be955e44c91877fce77b4b9daa3
2024-11-19T07:37:30,395 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/dbdbdbe46cbe4e55bca1494a7badbaf0 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/dbdbdbe46cbe4e55bca1494a7badbaf0
2024-11-19T07:37:30,396 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/73243dc926db4862bdd38548a9c908ff
2024-11-19T07:37:30,397 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/26efb6dbb360474682ebb6ee16495d82
2024-11-19T07:37:30,398 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/7c8dbcb421a14369a6fe0fd792c2c04b to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/7c8dbcb421a14369a6fe0fd792c2c04b
2024-11-19T07:37:30,399 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/00318f2c243f484084f36e13d3d09177
2024-11-19T07:37:30,400 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/d79595d49f5d41879c96b833c9d7e4c2
2024-11-19T07:37:30,401 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/c3ce67b1362442099688941e823bb0b3 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/c3ce67b1362442099688941e823bb0b3
2024-11-19T07:37:30,402 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/99fb7e67d46640a48a8c586e0fbc9e0c
2024-11-19T07:37:30,402 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/5060b84c115c4f6ca45e92460474063b
2024-11-19T07:37:30,403 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0483b689a35942078bb209ddedbe828c to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/0483b689a35942078bb209ddedbe828c
2024-11-19T07:37:30,404 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/75865ae56f354faabece6aa5ce4b2981
2024-11-19T07:37:30,405 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/681983dd3ec446caaebf7fe205837805
2024-11-19T07:37:30,406 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/79bc49229b534a28a4047d5b1be10e8c to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/79bc49229b534a28a4047d5b1be10e8c
2024-11-19T07:37:30,406 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/19b83451823942ce983f96ab9a8f6806
2024-11-19T07:37:30,407 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/4dcec10743e14263ad15613223ee726a
2024-11-19T07:37:30,408 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0 to hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/archive/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/info/6a97d652aa54483cafaf13424fd73bf0
2024-11-19T07:37:30,408 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1cba198dad964c4fa46471f9792e9ef9=8260, 0640bb125c2341d3b781078391e502d5=12509, 675b003f81a4427794d98bd87165cc6e=36426, de7c94e83832450c8f8553bfc9d13ea1=25453, 1c99ced9ca804d79ba42b70d780d7fcf=12510, be51359bfe984b02b85b7aa5fc488aad=55934, b6aab149b2564b659cdc147c546e6c6e=16828, 20b77be955e44c91877fce77b4b9daa3=16828, dbdbdbe46cbe4e55bca1494a7badbaf0=78811, 73243dc926db4862bdd38548a9c908ff=15750, 26efb6dbb360474682ebb6ee16495d82=16828, 7c8dbcb421a14369a6fe0fd792c2c04b=103711, 00318f2c243f484084f36e13d3d09177=17906, d79595d49f5d41879c96b833c9d7e4c2=14672, c3ce67b1362442099688941e823bb0b3=127665, 99fb7e67d46640a48a8c586e0fbc9e0c=19000, 5060b84c115c4f6ca45e92460474063b=16828, 0483b689a35942078bb209ddedbe828c=148322, 75865ae56f354faabece6aa5ce4b2981=13594, 681983dd3ec446caaebf7fe205837805=17918, 79bc49229b534a28a4047d5b1be10e8c=181881, 19b83451823942ce983f96ab9a8f6806=25491, 4dcec10743e14263ad15613223ee726a=12523, 6a97d652aa54483cafaf13424fd73bf0=16839]
2024-11-19T07:37:30,411 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/data/default/TestLogRolling-testLogRolling/d6b0d555b374134379e8a6f81b8ef210/recovered.edits/340.seqid, newMaxSeqId=340, maxSeqId=85
2024-11-19T07:37:30,411 INFO  [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:30,412 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6b0d555b374134379e8a6f81b8ef210:
Waiting for close lock at 1732001850382Running coprocessor pre-close hooks at 1732001850382Disabling compacts and flushes for region at 1732001850382Disabling writes for close at 1732001850383 (+1 ms)Writing region close event to WAL at 1732001850408 (+25 ms)Running coprocessor post-close hooks at 1732001850411 (+3 ms)Closed at 1732001850411
2024-11-19T07:37:30,412 DEBUG [RS_CLOSE_REGION-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732001825665.d6b0d555b374134379e8a6f81b8ef210.
2024-11-19T07:37:30,527 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-19T07:37:30,527 INFO  [regionserver/db0afee3eab9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-19T07:37:30,529 INFO  [regionserver/db0afee3eab9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:37:30,573 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,36609,1732001801933; all regions closed.
2024-11-19T07:37:30,574 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,574 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,574 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,574 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,575 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741834_1010 (size=8107)
2024-11-19T07:37:30,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741834_1010 (size=8107)
2024-11-19T07:37:30,581 DEBUG [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs
2024-11-19T07:37:30,581 INFO  [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C36609%2C1732001801933.meta:.meta(num 1732001803145)
2024-11-19T07:37:30,581 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,581 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,582 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,582 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,582 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741880_1056 (size=778)
2024-11-19T07:37:30,585 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741880_1056 (size=778)
2024-11-19T07:37:30,587 DEBUG [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/oldWALs
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C36609%2C1732001801933:(num 1732001850213)
2024-11-19T07:37:30,587 DEBUG [RS:0;db0afee3eab9:36609 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:37:30,587 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:37:30,587 INFO  [RS:0;db0afee3eab9:36609 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36609
2024-11-19T07:37:30,599 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,36609,1732001801933
2024-11-19T07:37:30,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:37:30,599 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:37:30,609 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,36609,1732001801933]
2024-11-19T07:37:30,705 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,36609,1732001801933 already deleted, retry=false
2024-11-19T07:37:30,705 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,36609,1732001801933 expired; onlineServers=0
2024-11-19T07:37:30,705 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,37997,1732001801772' *****
2024-11-19T07:37:30,705 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:37:30,705 INFO  [M:0;db0afee3eab9:37997 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:37:30,705 INFO  [M:0;db0afee3eab9:37997 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:37:30,705 DEBUG [M:0;db0afee3eab9:37997 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:37:30,705 DEBUG [M:0;db0afee3eab9:37997 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:37:30,706 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:37:30,706 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001802419 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001802419,5,FailOnTimeoutGroup]
2024-11-19T07:37:30,706 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001802419 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001802419,5,FailOnTimeoutGroup]
2024-11-19T07:37:30,706 INFO  [M:0;db0afee3eab9:37997 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:37:30,706 INFO  [M:0;db0afee3eab9:37997 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:37:30,706 DEBUG [M:0;db0afee3eab9:37997 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:37:30,706 INFO  [M:0;db0afee3eab9:37997 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:37:30,707 INFO  [M:0;db0afee3eab9:37997 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:37:30,707 ERROR [M:0;db0afee3eab9:37997 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36347,5,PEWorkerGroup]
2024-11-19T07:37:30,707 INFO  [M:0;db0afee3eab9:37997 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:37:30,707 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:37:30,710 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:30,710 INFO  [RS:0;db0afee3eab9:36609 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:37:30,710 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36609-0x101520621c80001, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:30,710 INFO  [RS:0;db0afee3eab9:36609 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,36609,1732001801933; zookeeper connection closed.
2024-11-19T07:37:30,711 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27058c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27058c7
2024-11-19T07:37:30,711 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:37:30,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:37:30,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:30,715 DEBUG [M:0;db0afee3eab9:37997 {}] zookeeper.ZKUtil(347): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:37:30,715 WARN  [M:0;db0afee3eab9:37997 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:37:30,717 INFO  [M:0;db0afee3eab9:37997 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/.lastflushedseqids
2024-11-19T07:37:30,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741881_1057 (size=228)
2024-11-19T07:37:30,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741881_1057 (size=228)
2024-11-19T07:37:30,727 INFO  [M:0;db0afee3eab9:37997 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:37:30,727 INFO  [M:0;db0afee3eab9:37997 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:37:30,727 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:37:30,728 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:30,728 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:30,728 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:37:30,728 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:30,728 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.35 KB
2024-11-19T07:37:30,746 DEBUG [M:0;db0afee3eab9:37997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e25061412c24c9295e7b5095d677087 is 82, key is hbase:meta,,1/info:regioninfo/1732001803169/Put/seqid=0
2024-11-19T07:37:30,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741882_1058 (size=5672)
2024-11-19T07:37:30,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741882_1058 (size=5672)
2024-11-19T07:37:30,750 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e25061412c24c9295e7b5095d677087
2024-11-19T07:37:30,774 DEBUG [M:0;db0afee3eab9:37997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e8e50c128448c49254af3ac3bece2e is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732001803653/Put/seqid=0
2024-11-19T07:37:30,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741883_1059 (size=7090)
2024-11-19T07:37:30,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741883_1059 (size=7090)
2024-11-19T07:37:30,779 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e8e50c128448c49254af3ac3bece2e
2024-11-19T07:37:30,782 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 09e8e50c128448c49254af3ac3bece2e
2024-11-19T07:37:30,795 DEBUG [M:0;db0afee3eab9:37997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba03683c4197436e81fb6393f71f1e1a is 69, key is db0afee3eab9,36609,1732001801933/rs:state/1732001802504/Put/seqid=0
2024-11-19T07:37:30,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741884_1060 (size=5156)
2024-11-19T07:37:30,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741884_1060 (size=5156)
2024-11-19T07:37:30,800 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba03683c4197436e81fb6393f71f1e1a
2024-11-19T07:37:30,821 DEBUG [M:0;db0afee3eab9:37997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1c42c9b0f8b343989371d85329d32665 is 52, key is load_balancer_on/state:d/1732001803275/Put/seqid=0
2024-11-19T07:37:30,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741885_1061 (size=5056)
2024-11-19T07:37:30,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741885_1061 (size=5056)
2024-11-19T07:37:30,826 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1c42c9b0f8b343989371d85329d32665
2024-11-19T07:37:30,831 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9e25061412c24c9295e7b5095d677087 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9e25061412c24c9295e7b5095d677087
2024-11-19T07:37:30,835 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9e25061412c24c9295e7b5095d677087, entries=8, sequenceid=125, filesize=5.5 K
2024-11-19T07:37:30,836 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e8e50c128448c49254af3ac3bece2e as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09e8e50c128448c49254af3ac3bece2e
2024-11-19T07:37:30,840 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 09e8e50c128448c49254af3ac3bece2e
2024-11-19T07:37:30,841 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09e8e50c128448c49254af3ac3bece2e, entries=13, sequenceid=125, filesize=6.9 K
2024-11-19T07:37:30,842 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba03683c4197436e81fb6393f71f1e1a as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ba03683c4197436e81fb6393f71f1e1a
2024-11-19T07:37:30,845 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ba03683c4197436e81fb6393f71f1e1a, entries=1, sequenceid=125, filesize=5.0 K
2024-11-19T07:37:30,846 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1c42c9b0f8b343989371d85329d32665 as hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1c42c9b0f8b343989371d85329d32665
2024-11-19T07:37:30,851 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36347/user/jenkins/test-data/91bfc930-756b-9b70-446f-7b615e025f74/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1c42c9b0f8b343989371d85329d32665, entries=1, sequenceid=125, filesize=4.9 K
2024-11-19T07:37:30,852 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=125, compaction requested=false
2024-11-19T07:37:30,855 INFO  [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:30,855 DEBUG [M:0;db0afee3eab9:37997 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001850727Disabling compacts and flushes for region at 1732001850727Disabling writes for close at 1732001850728 (+1 ms)Obtaining lock to block concurrent updates at 1732001850728Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001850728Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1732001850728Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001850729 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001850729Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001850746 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001850746Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001850755 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001850773 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001850773Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001850782 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001850795 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001850795Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001850804 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001850821 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001850821Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb507dc: reopening flushed file at 1732001850830 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@121065a9: reopening flushed file at 1732001850835 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f15ea9: reopening flushed file at 1732001850841 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f0c038: reopening flushed file at 1732001850846 (+5 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=125, compaction requested=false at 1732001850852 (+6 ms)Writing region close event to WAL at 1732001850855 (+3 ms)Closed at 1732001850855
2024-11-19T07:37:30,856 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,856 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,856 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,856 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,856 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:30,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741830_1006 (size=61320)
2024-11-19T07:37:30,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34561 is added to blk_1073741830_1006 (size=61320)
2024-11-19T07:37:30,858 INFO  [M:0;db0afee3eab9:37997 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:37:30,858 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:37:30,858 INFO  [M:0;db0afee3eab9:37997 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37997
2024-11-19T07:37:30,859 INFO  [M:0;db0afee3eab9:37997 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:37:30,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:30,968 INFO  [M:0;db0afee3eab9:37997 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:37:30,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37997-0x101520621c80000, quorum=127.0.0.1:56379, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:30,972 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ced388d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:30,973 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30ce08b8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:30,973 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:30,973 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dea9c62{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:30,973 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2fc2e7d1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:30,977 WARN  [BP-2122081010-172.17.0.2-1732001799482 heartbeating to localhost/127.0.0.1:36347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:37:30,977 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:37:30,977 WARN  [BP-2122081010-172.17.0.2-1732001799482 heartbeating to localhost/127.0.0.1:36347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122081010-172.17.0.2-1732001799482 (Datanode Uuid edf25135-b238-48f5-8014-b4eca792342e) service to localhost/127.0.0.1:36347
2024-11-19T07:37:30,977 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:37:30,977 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data3/current/BP-2122081010-172.17.0.2-1732001799482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:30,978 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data4/current/BP-2122081010-172.17.0.2-1732001799482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:30,978 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:37:30,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77f3cd08{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:30,980 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2401b6df{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:30,980 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:30,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b95b0ea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:30,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@60b9b83d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:30,982 WARN  [BP-2122081010-172.17.0.2-1732001799482 heartbeating to localhost/127.0.0.1:36347 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:37:30,982 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:37:30,982 WARN  [BP-2122081010-172.17.0.2-1732001799482 heartbeating to localhost/127.0.0.1:36347 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122081010-172.17.0.2-1732001799482 (Datanode Uuid 05167d0f-86cf-420d-9838-091778010c66) service to localhost/127.0.0.1:36347
2024-11-19T07:37:30,982 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:37:30,983 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data1/current/BP-2122081010-172.17.0.2-1732001799482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:30,983 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/cluster_09ddcefc-40f6-b4c8-a5c1-89212524545c/data/data2/current/BP-2122081010-172.17.0.2-1732001799482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:30,983 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:37:30,989 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4328dba6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:37:30,990 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1dab95de{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:30,990 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:30,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3def21d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:30,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75966949{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:30,997 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:37:31,029 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:37:31,036 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 206)
Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36347
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36347
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36347 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36347
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36347 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36347
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36347
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36347 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:36347
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=521 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=147 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=12019 (was 11233) - AvailableMemoryMB LEAK? -
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=521, MaxFileDescriptor=1048576, SystemLoadAverage=147, ProcessCount=11, AvailableMemoryMB=12019
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.log.dir so I do NOT create it in target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/056d1a7a-617f-9114-0fc0-65c1ada37bdc/hadoop.tmp.dir so I do NOT create it in target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11, deleteOnExit=true
2024-11-19T07:37:31,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/test.cache.data in system properties and HBase conf
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.tmp.dir in system properties and HBase conf
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir in system properties and HBase conf
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-19T07:37:31,044 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-19T07:37:31,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/nfs.dump.dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/java.io.tmpdir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-19T07:37:31,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-19T07:37:31,058 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:37:31,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:37:31,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-19T07:37:31,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-19T07:37:31,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-19T07:37:31,178 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:31,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:31,438 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:37:31,441 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:37:31,443 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:37:31,443 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:37:31,443 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:37:31,443 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:37:31,444 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3698cd86{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:37:31,444 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b825638{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:37:31,535 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@71465f4c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/java.io.tmpdir/jetty-localhost-39723-hadoop-hdfs-3_4_1-tests_jar-_-any-1887195078056119159/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:37:31,535 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2191d18b{HTTP/1.1, (http/1.1)}{localhost:39723}
2024-11-19T07:37:31,536 INFO  [Time-limited test {}] server.Server(415): Started @308224ms
2024-11-19T07:37:31,546 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-19T07:37:31,783 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:37:31,785 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:37:31,786 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:37:31,786 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:37:31,786 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-19T07:37:31,787 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75728f34{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:37:31,787 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ecbf0f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:37:31,840 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=9, created chunk count=9, reused chunk count=73, reuseRatio=89.02%
2024-11-19T07:37:31,841 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-19T07:37:31,878 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@509dedeb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/java.io.tmpdir/jetty-localhost-36033-hadoop-hdfs-3_4_1-tests_jar-_-any-12872656267932174934/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:31,878 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@46761010{HTTP/1.1, (http/1.1)}{localhost:36033}
2024-11-19T07:37:31,879 INFO  [Time-limited test {}] server.Server(415): Started @308567ms
2024-11-19T07:37:31,879 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:37:31,904 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-19T07:37:31,906 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-19T07:37:31,907 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-19T07:37:31,907 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-19T07:37:31,907 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-19T07:37:31,907 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@477d322e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,AVAILABLE}
2024-11-19T07:37:31,908 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@753778e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-19T07:37:31,998 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@73bc7eb2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/java.io.tmpdir/jetty-localhost-46547-hadoop-hdfs-3_4_1-tests_jar-_-any-8026648017478074240/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:31,998 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@624ed4c3{HTTP/1.1, (http/1.1)}{localhost:46547}
2024-11-19T07:37:31,998 INFO  [Time-limited test {}] server.Server(415): Started @308687ms
2024-11-19T07:37:31,999 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-19T07:37:32,179 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:32,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:33,068 WARN  [Thread-2512 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data1/current/BP-1830346352-172.17.0.2-1732001851060/current, will proceed with Du for space computation calculation, 
2024-11-19T07:37:33,068 WARN  [Thread-2513 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data2/current/BP-1830346352-172.17.0.2-1732001851060/current, will proceed with Du for space computation calculation, 
2024-11-19T07:37:33,084 WARN  [Thread-2476 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:37:33,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6227ff3ba209c2a with lease ID 0x5af95639f73257f6: Processing first storage report for DS-5ccc60b7-c639-419c-a885-4f279e8ab971 from datanode DatanodeRegistration(127.0.0.1:44221, datanodeUuid=7b3f1de5-3822-451d-81e2-dca5cec8acda, infoPort=45681, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060)
2024-11-19T07:37:33,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6227ff3ba209c2a with lease ID 0x5af95639f73257f6: from storage DS-5ccc60b7-c639-419c-a885-4f279e8ab971 node DatanodeRegistration(127.0.0.1:44221, datanodeUuid=7b3f1de5-3822-451d-81e2-dca5cec8acda, infoPort=45681, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:37:33,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6227ff3ba209c2a with lease ID 0x5af95639f73257f6: Processing first storage report for DS-3174a129-d128-46f9-9b4f-09f817de6868 from datanode DatanodeRegistration(127.0.0.1:44221, datanodeUuid=7b3f1de5-3822-451d-81e2-dca5cec8acda, infoPort=45681, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060)
2024-11-19T07:37:33,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6227ff3ba209c2a with lease ID 0x5af95639f73257f6: from storage DS-3174a129-d128-46f9-9b4f-09f817de6868 node DatanodeRegistration(127.0.0.1:44221, datanodeUuid=7b3f1de5-3822-451d-81e2-dca5cec8acda, infoPort=45681, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:37:33,180 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:33,195 WARN  [Thread-2523 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data3/current/BP-1830346352-172.17.0.2-1732001851060/current, will proceed with Du for space computation calculation, 
2024-11-19T07:37:33,195 WARN  [Thread-2524 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data4/current/BP-1830346352-172.17.0.2-1732001851060/current, will proceed with Du for space computation calculation, 
2024-11-19T07:37:33,216 WARN  [Thread-2499 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-19T07:37:33,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1027f4293b5ab77d with lease ID 0x5af95639f73257f7: Processing first storage report for DS-280ee53c-5080-4f04-8e43-11041151953b from datanode DatanodeRegistration(127.0.0.1:44629, datanodeUuid=30644da4-ae01-4ef6-8fa2-267bf45e58c1, infoPort=32773, infoSecurePort=0, ipcPort=41535, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060)
2024-11-19T07:37:33,218 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1027f4293b5ab77d with lease ID 0x5af95639f73257f7: from storage DS-280ee53c-5080-4f04-8e43-11041151953b node DatanodeRegistration(127.0.0.1:44629, datanodeUuid=30644da4-ae01-4ef6-8fa2-267bf45e58c1, infoPort=32773, infoSecurePort=0, ipcPort=41535, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:37:33,219 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1027f4293b5ab77d with lease ID 0x5af95639f73257f7: Processing first storage report for DS-c0b35c8a-dee0-4cf2-9dcf-6d353396241c from datanode DatanodeRegistration(127.0.0.1:44629, datanodeUuid=30644da4-ae01-4ef6-8fa2-267bf45e58c1, infoPort=32773, infoSecurePort=0, ipcPort=41535, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060)
2024-11-19T07:37:33,219 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1027f4293b5ab77d with lease ID 0x5af95639f73257f7: from storage DS-c0b35c8a-dee0-4cf2-9dcf-6d353396241c node DatanodeRegistration(127.0.0.1:44629, datanodeUuid=30644da4-ae01-4ef6-8fa2-267bf45e58c1, infoPort=32773, infoSecurePort=0, ipcPort=41535, storageInfo=lv=-57;cid=testClusterID;nsid=1566495781;c=1732001851060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-19T07:37:33,227 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71
2024-11-19T07:37:33,233 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/zookeeper_0, clientPort=59789, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-19T07:37:33,234 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59789
2024-11-19T07:37:33,234 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,235 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:37:33,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741825_1001 (size=7)
2024-11-19T07:37:33,245 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d with version=8
2024-11-19T07:37:33,245 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:45255/user/jenkins/test-data/db33cf86-90bb-a299-dacb-af73656a892b/hbase-staging
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-19T07:37:33,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:37:33,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32791
2024-11-19T07:37:33,249 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32791 connecting to ZooKeeper ensemble=127.0.0.1:59789
2024-11-19T07:37:33,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:327910x0, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:37:33,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32791-0x1015206ead90000 connected
2024-11-19T07:37:33,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:33,389 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,393 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:33,397 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d, hbase.cluster.distributed=false
2024-11-19T07:37:33,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:37:33,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32791
2024-11-19T07:37:33,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32791
2024-11-19T07:37:33,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32791
2024-11-19T07:37:33,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32791
2024-11-19T07:37:33,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32791
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/db0afee3eab9:0 server-side Connection retries=45
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-19T07:37:33,418 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42783
2024-11-19T07:37:33,419 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42783 connecting to ZooKeeper ensemble=127.0.0.1:59789
2024-11-19T07:37:33,420 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,421 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427830x0, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-19T07:37:33,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42783-0x1015206ead90001 connected
2024-11-19T07:37:33,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:33,436 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-19T07:37:33,437 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-19T07:37:33,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-19T07:37:33,439 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-19T07:37:33,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42783
2024-11-19T07:37:33,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42783
2024-11-19T07:37:33,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42783
2024-11-19T07:37:33,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42783
2024-11-19T07:37:33,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42783
2024-11-19T07:37:33,451 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db0afee3eab9:32791
2024-11-19T07:37:33,451 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:37:33,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:37:33,462 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-19T07:37:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,473 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-19T07:37:33,474 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db0afee3eab9,32791,1732001853247 from backup master directory
2024-11-19T07:37:33,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:37:33,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-19T07:37:33,483 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:37:33,483 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,490 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/hbase.id] with ID: 109b2724-551b-4e49-a4ea-8eb122733aca
2024-11-19T07:37:33,490 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/.tmp/hbase.id
2024-11-19T07:37:33,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:37:33,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741826_1002 (size=42)
2024-11-19T07:37:33,499 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/.tmp/hbase.id]:[hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/hbase.id]
2024-11-19T07:37:33,516 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:33,516 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-19T07:37:33,518 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-19T07:37:33,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:37:33,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741827_1003 (size=196)
2024-11-19T07:37:33,537 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-19T07:37:33,538 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-19T07:37:33,538 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:37:33,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:37:33,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741828_1004 (size=1189)
2024-11-19T07:37:33,548 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store
2024-11-19T07:37:33,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:37:33,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741829_1005 (size=34)
2024-11-19T07:37:33,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:33,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:37:33,555 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:33,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:33,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:37:33,555 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:33,556 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:33,556 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001853555Disabling compacts and flushes for region at 1732001853555Disabling writes for close at 1732001853555Writing region close event to WAL at 1732001853555Closed at 1732001853555
2024-11-19T07:37:33,556 WARN  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/.initializing
2024-11-19T07:37:33,556 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/WALs/db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,559 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C32791%2C1732001853247, suffix=, logDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/WALs/db0afee3eab9,32791,1732001853247, archiveDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/oldWALs, maxLogs=10
2024-11-19T07:37:33,559 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C32791%2C1732001853247.1732001853559
2024-11-19T07:37:33,564 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/WALs/db0afee3eab9,32791,1732001853247/db0afee3eab9%2C32791%2C1732001853247.1732001853559
2024-11-19T07:37:33,565 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32773:32773),(127.0.0.1/127.0.0.1:45681:45681)]
2024-11-19T07:37:33,566 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:37:33,566 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:33,566 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,566 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,568 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:37:33,569 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-19T07:37:33,569 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:33,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:37:33,571 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-19T07:37:33,571 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:37:33,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:37:33,573 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-19T07:37:33,573 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:37:33,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-19T07:37:33,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-19T07:37:33,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-19T07:37:33,576 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,577 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,578 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,580 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,580 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,580 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-19T07:37:33,582 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-19T07:37:33,584 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:37:33,585 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=840064, jitterRate=0.06819750368595123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-19T07:37:33,585 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732001853566Initializing all the Stores at 1732001853567 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001853567Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001853568 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001853568Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001853568Cleaning up temporary data from old regions at 1732001853580 (+12 ms)Region opened successfully at 1732001853585 (+5 ms)
2024-11-19T07:37:33,586 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-19T07:37:33,589 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@466f3d82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:37:33,590 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-19T07:37:33,590 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-19T07:37:33,591 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-19T07:37:33,591 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-19T07:37:33,591 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-19T07:37:33,592 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-19T07:37:33,592 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-19T07:37:33,594 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-19T07:37:33,595 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-19T07:37:33,604 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-19T07:37:33,604 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-19T07:37:33,605 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-19T07:37:33,614 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-19T07:37:33,615 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-19T07:37:33,616 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-19T07:37:33,625 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-19T07:37:33,627 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-19T07:37:33,636 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-19T07:37:33,640 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-19T07:37:33,651 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-19T07:37:33,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:33,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:33,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,663 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db0afee3eab9,32791,1732001853247, sessionid=0x1015206ead90000, setting cluster-up flag (Was=false)
2024-11-19T07:37:33,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,889 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-19T07:37:33,892 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:33,947 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-19T07:37:33,951 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db0afee3eab9,32791,1732001853247
2024-11-19T07:37:33,954 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-19T07:37:33,958 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-19T07:37:33,958 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-19T07:37:33,958 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-19T07:37:33,959 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db0afee3eab9,32791,1732001853247
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db0afee3eab9:0, corePoolSize=5, maxPoolSize=5
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db0afee3eab9:0, corePoolSize=10, maxPoolSize=10
2024-11-19T07:37:33,960 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:33,961 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:37:33,961 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732001883962
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:33,962 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:37:33,962 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-19T07:37:33,962 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-19T07:37:33,963 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001853963,5,FailOnTimeoutGroup]
2024-11-19T07:37:33,963 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,963 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001853963,5,FailOnTimeoutGroup]
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:33,963 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:33,963 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-19T07:37:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:37:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741831_1007 (size=1321)
2024-11-19T07:37:33,971 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-19T07:37:33,971 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d
2024-11-19T07:37:33,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:37:33,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741832_1008 (size=32)
2024-11-19T07:37:33,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:33,979 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:37:33,980 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:37:33,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,980 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:33,980 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:37:33,981 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:37:33,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:33,981 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:37:33,982 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:37:33,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,982 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:33,982 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:37:33,983 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:37:33,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:33,983 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:33,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:37:33,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740
2024-11-19T07:37:33,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740
2024-11-19T07:37:33,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:37:33,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:37:33,986 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:37:33,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:37:33,989 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-19T07:37:33,989 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=722856, jitterRate=-0.08084139227867126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732001853978Initializing all the Stores at 1732001853978Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001853979 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001853979Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001853979Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001853979Cleaning up temporary data from old regions at 1732001853986 (+7 ms)Region opened successfully at 1732001853990 (+4 ms)
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:37:33,990 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:37:33,990 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:37:33,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001853990Disabling compacts and flushes for region at 1732001853990Disabling writes for close at 1732001853990Writing region close event to WAL at 1732001853990Closed at 1732001853990
2024-11-19T07:37:33,991 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:37:33,992 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-19T07:37:33,992 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-19T07:37:33,993 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:37:33,994 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-19T07:37:34,042 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(746): ClusterId : 109b2724-551b-4e49-a4ea-8eb122733aca
2024-11-19T07:37:34,043 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-19T07:37:34,053 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-19T07:37:34,053 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-19T07:37:34,063 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-19T07:37:34,064 DEBUG [RS:0;db0afee3eab9:42783 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be4b9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db0afee3eab9/172.17.0.2:0
2024-11-19T07:37:34,078 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db0afee3eab9:42783
2024-11-19T07:37:34,078 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-19T07:37:34,078 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-19T07:37:34,078 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-19T07:37:34,079 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(2659): reportForDuty to master=db0afee3eab9,32791,1732001853247 with port=42783, startcode=1732001853417
2024-11-19T07:37:34,079 DEBUG [RS:0;db0afee3eab9:42783 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-19T07:37:34,081 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60303, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-19T07:37:34,081 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32791 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,081 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32791 {}] master.ServerManager(517): Registering regionserver=db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,082 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d
2024-11-19T07:37:34,082 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45495
2024-11-19T07:37:34,082 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-19T07:37:34,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:37:34,094 DEBUG [RS:0;db0afee3eab9:42783 {}] zookeeper.ZKUtil(111): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,094 WARN  [RS:0;db0afee3eab9:42783 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-19T07:37:34,094 INFO  [RS:0;db0afee3eab9:42783 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:37:34,094 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,094 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db0afee3eab9,42783,1732001853417]
2024-11-19T07:37:34,097 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-19T07:37:34,098 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-19T07:37:34,099 INFO  [RS:0;db0afee3eab9:42783 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-19T07:37:34,099 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,099 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-19T07:37:34,099 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-19T07:37:34,099 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db0afee3eab9:0, corePoolSize=2, maxPoolSize=2
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db0afee3eab9:0, corePoolSize=1, maxPoolSize=1
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:37:34,100 DEBUG [RS:0;db0afee3eab9:42783 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db0afee3eab9:0, corePoolSize=3, maxPoolSize=3
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,104 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42783,1732001853417-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:37:34,117 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-19T07:37:34,117 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,42783,1732001853417-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,117 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,117 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.Replication(171): db0afee3eab9,42783,1732001853417 started
2024-11-19T07:37:34,129 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,129 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1482): Serving as db0afee3eab9,42783,1732001853417, RpcServer on db0afee3eab9/172.17.0.2:42783, sessionid=0x1015206ead90001
2024-11-19T07:37:34,129 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-19T07:37:34,129 DEBUG [RS:0;db0afee3eab9:42783 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,129 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42783,1732001853417'
2024-11-19T07:37:34,129 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db0afee3eab9,42783,1732001853417'
2024-11-19T07:37:34,130 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-19T07:37:34,131 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-19T07:37:34,131 DEBUG [RS:0;db0afee3eab9:42783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-19T07:37:34,131 INFO  [RS:0;db0afee3eab9:42783 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-19T07:37:34,131 INFO  [RS:0;db0afee3eab9:42783 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-19T07:37:34,144 WARN  [db0afee3eab9:32791 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-19T07:37:34,181 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:34,235 INFO  [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42783%2C1732001853417, suffix=, logDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/db0afee3eab9,42783,1732001853417, archiveDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs, maxLogs=32
2024-11-19T07:37:34,237 INFO  [RS:0;db0afee3eab9:42783 {}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42783%2C1732001853417.1732001854236
2024-11-19T07:37:34,244 INFO  [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/db0afee3eab9,42783,1732001853417/db0afee3eab9%2C42783%2C1732001853417.1732001854236
2024-11-19T07:37:34,246 DEBUG [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32773:32773),(127.0.0.1/127.0.0.1:45681:45681)]
2024-11-19T07:37:34,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.1732001656802
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:34,394 DEBUG [db0afee3eab9:32791 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-19T07:37:34,395 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,399 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42783,1732001853417, state=OPENING
2024-11-19T07:37:34,431 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-19T07:37:34,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:34,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:34,443 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-19T07:37:34,444 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:37:34,444 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:37:34,444 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42783,1732001853417}]
2024-11-19T07:37:34,600 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-19T07:37:34,603 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38971, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-19T07:37:34,609 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-19T07:37:34,609 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:37:34,611 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db0afee3eab9%2C42783%2C1732001853417.meta, suffix=.meta, logDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/db0afee3eab9,42783,1732001853417, archiveDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs, maxLogs=32
2024-11-19T07:37:34,612 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor db0afee3eab9%2C42783%2C1732001853417.meta.1732001854612.meta
2024-11-19T07:37:34,618 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/db0afee3eab9,42783,1732001853417/db0afee3eab9%2C42783%2C1732001853417.meta.1732001854612.meta
2024-11-19T07:37:34,619 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32773:32773),(127.0.0.1/127.0.0.1:45681:45681)]
2024-11-19T07:37:34,620 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-19T07:37:34,620 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-19T07:37:34,620 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-19T07:37:34,620 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-19T07:37:34,620 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-19T07:37:34,621 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-19T07:37:34,621 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-19T07:37:34,621 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-19T07:37:34,622 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-19T07:37:34,623 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-19T07:37:34,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:34,623 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:34,623 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-19T07:37:34,624 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-19T07:37:34,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:34,624 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:34,624 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-19T07:37:34,625 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-19T07:37:34,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:34,625 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:34,625 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-19T07:37:34,626 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-19T07:37:34,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-19T07:37:34,626 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-19T07:37:34,626 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-19T07:37:34,627 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740
2024-11-19T07:37:34,627 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740
2024-11-19T07:37:34,628 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-19T07:37:34,629 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-19T07:37:34,629 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-19T07:37:34,630 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-19T07:37:34,631 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780927, jitterRate=-0.007000148296356201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-19T07:37:34,631 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-19T07:37:34,631 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732001854621Writing region info on filesystem at 1732001854621Initializing all the Stores at 1732001854621Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001854622 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001854622Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732001854622Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732001854622Cleaning up temporary data from old regions at 1732001854629 (+7 ms)Running coprocessor post-open hooks at 1732001854631 (+2 ms)Region opened successfully at 1732001854631
2024-11-19T07:37:34,632 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732001854600
2024-11-19T07:37:34,634 DEBUG [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-19T07:37:34,634 INFO  [RS_OPEN_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-19T07:37:34,635 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,636 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db0afee3eab9,42783,1732001853417, state=OPEN
2024-11-19T07:37:34,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:37:34,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-19T07:37:34,676 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:37:34,676 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-19T07:37:34,678 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-19T07:37:34,678 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db0afee3eab9,42783,1732001853417 in 232 msec
2024-11-19T07:37:34,679 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-19T07:37:34,679 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 686 msec
2024-11-19T07:37:34,680 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-19T07:37:34,680 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-19T07:37:34,681 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:37:34,681 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42783,1732001853417, seqNum=-1]
2024-11-19T07:37:34,681 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:37:34,682 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55577, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:37:34,687 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 730 msec
2024-11-19T07:37:34,687 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732001854687, completionTime=-1
2024-11-19T07:37:34,687 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-19T07:37:34,687 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732001914689
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732001974689
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db0afee3eab9:32791, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,689 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,690 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,691 DEBUG [master/db0afee3eab9:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.209sec
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-19T07:37:34,693 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-19T07:37:34,695 DEBUG [master/db0afee3eab9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-19T07:37:34,695 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-19T07:37:34,695 INFO  [master/db0afee3eab9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db0afee3eab9,32791,1732001853247-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-19T07:37:34,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4756e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:37:34,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db0afee3eab9,32791,-1 for getting cluster id
2024-11-19T07:37:34,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-19T07:37:34,746 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '109b2724-551b-4e49-a4ea-8eb122733aca'
2024-11-19T07:37:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-19T07:37:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "109b2724-551b-4e49-a4ea-8eb122733aca"

2024-11-19T07:37:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6520bcfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:37:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db0afee3eab9,32791,-1]
2024-11-19T07:37:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-19T07:37:34,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:34,749 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-19T07:37:34,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158d184, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-19T07:37:34,751 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-19T07:37:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db0afee3eab9,42783,1732001853417, seqNum=-1]
2024-11-19T07:37:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-19T07:37:34,753 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-19T07:37:34,755 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db0afee3eab9,32791,1732001853247
2024-11-19T07:37:34,755 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-19T07:37:34,757 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-19T07:37:34,757 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-19T07:37:34,759 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1, archiveDir=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs, maxLogs=32
2024-11-19T07:37:34,760 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732001854759
2024-11-19T07:37:34,764 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1/test.com%2C8080%2C1.1732001854759
2024-11-19T07:37:34,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32773:32773),(127.0.0.1/127.0.0.1:45681:45681)]
2024-11-19T07:37:34,766 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732001854766
2024-11-19T07:37:34,770 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,770 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,770 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,770 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,771 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,771 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1/test.com%2C8080%2C1.1732001854759 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1/test.com%2C8080%2C1.1732001854766
2024-11-19T07:37:34,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45681:45681),(127.0.0.1/127.0.0.1:32773:32773)]
2024-11-19T07:37:34,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1/test.com%2C8080%2C1.1732001854759 is not closed yet, will try archiving it next time
2024-11-19T07:37:34,772 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741835_1011 (size=93)
2024-11-19T07:37:34,772 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,773 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741835_1011 (size=93)
2024-11-19T07:37:34,773 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,773 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:34,774 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/WALs/test.com,8080,1/test.com%2C8080%2C1.1732001854759 to hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs/test.com%2C8080%2C1.1732001854759
2024-11-19T07:37:34,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741836_1012 (size=93)
2024-11-19T07:37:34,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741836_1012 (size=93)
2024-11-19T07:37:34,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs
2024-11-19T07:37:34,777 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732001854766)
2024-11-19T07:37:34,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-19T07:37:34,777 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:37:34,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:34,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:34,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:34,778 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-19T07:37:34,778 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-19T07:37:34,778 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=986419994, stopped=false
2024-11-19T07:37:34,778 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db0afee3eab9,32791,1732001853247
2024-11-19T07:37:34,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:34,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-19T07:37:34,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:34,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:34,799 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:37:34,799 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-19T07:37:34,800 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:34,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:34,800 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db0afee3eab9,42783,1732001853417' *****
2024-11-19T07:37:34,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:34,800 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-19T07:37:34,800 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-19T07:37:34,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-19T07:37:34,801 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(959): stopping server db0afee3eab9,42783,1732001853417
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db0afee3eab9:42783.
2024-11-19T07:37:34,801 DEBUG [RS:0;db0afee3eab9:42783 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-19T07:37:34,801 DEBUG [RS:0;db0afee3eab9:42783 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-19T07:37:34,801 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-19T07:37:34,802 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-19T07:37:34,802 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-19T07:37:34,802 DEBUG [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-19T07:37:34,802 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-19T07:37:34,802 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-19T07:37:34,802 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-19T07:37:34,802 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-19T07:37:34,802 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-19T07:37:34,802 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-19T07:37:34,819 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/.tmp/ns/65bbf76e427b44f9b0a981492a7e5424 is 43, key is default/ns:d/1732001854683/Put/seqid=0
2024-11-19T07:37:34,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741837_1013 (size=5153)
2024-11-19T07:37:34,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741837_1013 (size=5153)
2024-11-19T07:37:34,824 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/.tmp/ns/65bbf76e427b44f9b0a981492a7e5424
2024-11-19T07:37:34,828 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/.tmp/ns/65bbf76e427b44f9b0a981492a7e5424 as hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/ns/65bbf76e427b44f9b0a981492a7e5424
2024-11-19T07:37:34,832 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/ns/65bbf76e427b44f9b0a981492a7e5424, entries=2, sequenceid=6, filesize=5.0 K
2024-11-19T07:37:34,833 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false
2024-11-19T07:37:34,833 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-19T07:37:34,838 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-19T07:37:34,838 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-19T07:37:34,838 INFO  [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-19T07:37:34,838 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732001854802Running coprocessor pre-close hooks at 1732001854802Disabling compacts and flushes for region at 1732001854802Disabling writes for close at 1732001854802Obtaining lock to block concurrent updates at 1732001854802Preparing flush snapshotting stores in 1588230740 at 1732001854802Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732001854803 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732001854804 (+1 ms)Flushing 1588230740/ns: creating writer at 1732001854804Flushing 1588230740/ns: appending metadata at 1732001854818 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732001854819 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b7a72b: reopening flushed file at 1732001854828 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1732001854833 (+5 ms)Writing region close event to WAL at 1732001854834 (+1 ms)Running coprocessor post-close hooks at 1732001854838 (+4 ms)Closed at 1732001854838
2024-11-19T07:37:34,838 DEBUG [RS_CLOSE_META-regionserver/db0afee3eab9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-19T07:37:35,002 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(976): stopping server db0afee3eab9,42783,1732001853417; all regions closed.
2024-11-19T07:37:35,004 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,004 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,004 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,005 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,005 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741834_1010 (size=1152)
2024-11-19T07:37:35,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741834_1010 (size=1152)
2024-11-19T07:37:35,015 DEBUG [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs
2024-11-19T07:37:35,015 INFO  [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C42783%2C1732001853417.meta:.meta(num 1732001854612)
2024-11-19T07:37:35,016 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,016 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,016 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,016 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,016 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741833_1009 (size=93)
2024-11-19T07:37:35,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741833_1009 (size=93)
2024-11-19T07:37:35,019 DEBUG [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/oldWALs
2024-11-19T07:37:35,019 INFO  [RS:0;db0afee3eab9:42783 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog db0afee3eab9%2C42783%2C1732001853417:(num 1732001854236)
2024-11-19T07:37:35,019 DEBUG [RS:0;db0afee3eab9:42783 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-19T07:37:35,019 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.LeaseManager(133): Closed leases
2024-11-19T07:37:35,019 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:37:35,019 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.ChoreService(370): Chore service for: regionserver/db0afee3eab9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-19T07:37:35,020 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:37:35,020 INFO  [regionserver/db0afee3eab9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:37:35,020 INFO  [RS:0;db0afee3eab9:42783 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42783
2024-11-19T07:37:35,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-19T07:37:35,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db0afee3eab9,42783,1732001853417
2024-11-19T07:37:35,030 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:37:35,041 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db0afee3eab9,42783,1732001853417]
2024-11-19T07:37:35,051 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db0afee3eab9,42783,1732001853417 already deleted, retry=false
2024-11-19T07:37:35,051 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db0afee3eab9,42783,1732001853417 expired; onlineServers=0
2024-11-19T07:37:35,051 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db0afee3eab9,32791,1732001853247' *****
2024-11-19T07:37:35,051 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-19T07:37:35,052 DEBUG [M:0;db0afee3eab9:32791 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-19T07:37:35,052 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-19T07:37:35,052 DEBUG [M:0;db0afee3eab9:32791 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-19T07:37:35,052 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001853963 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.large.0-1732001853963,5,FailOnTimeoutGroup]
2024-11-19T07:37:35,052 DEBUG [master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001853963 {}] cleaner.HFileCleaner(306): Exit Thread[master/db0afee3eab9:0:becomeActiveMaster-HFileCleaner.small.0-1732001853963,5,FailOnTimeoutGroup]
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] hbase.ChoreService(370): Chore service for: master/db0afee3eab9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-19T07:37:35,052 DEBUG [M:0;db0afee3eab9:32791 {}] master.HMaster(1795): Stopping service threads
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-19T07:37:35,052 INFO  [M:0;db0afee3eab9:32791 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-19T07:37:35,053 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-19T07:37:35,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-19T07:37:35,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-19T07:37:35,062 DEBUG [M:0;db0afee3eab9:32791 {}] zookeeper.ZKUtil(347): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-19T07:37:35,062 WARN  [M:0;db0afee3eab9:32791 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-19T07:37:35,063 INFO  [M:0;db0afee3eab9:32791 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/.lastflushedseqids
2024-11-19T07:37:35,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741838_1014 (size=99)
2024-11-19T07:37:35,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741838_1014 (size=99)
2024-11-19T07:37:35,071 INFO  [M:0;db0afee3eab9:32791 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-19T07:37:35,071 INFO  [M:0;db0afee3eab9:32791 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-19T07:37:35,072 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-19T07:37:35,072 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:35,072 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:35,072 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-19T07:37:35,072 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:35,072 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-19T07:37:35,090 DEBUG [M:0;db0afee3eab9:32791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b6ccc2e7cd24f45ba6b5c443428dff8 is 82, key is hbase:meta,,1/info:regioninfo/1732001854635/Put/seqid=0
2024-11-19T07:37:35,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741839_1015 (size=5672)
2024-11-19T07:37:35,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741839_1015 (size=5672)
2024-11-19T07:37:35,095 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b6ccc2e7cd24f45ba6b5c443428dff8
2024-11-19T07:37:35,112 DEBUG [M:0;db0afee3eab9:32791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977eb86f92f248e5b3b7ca5f08b38d91 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732001854686/Put/seqid=0
2024-11-19T07:37:35,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741840_1016 (size=5275)
2024-11-19T07:37:35,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741840_1016 (size=5275)
2024-11-19T07:37:35,116 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977eb86f92f248e5b3b7ca5f08b38d91
2024-11-19T07:37:35,132 DEBUG [M:0;db0afee3eab9:32791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4137432ffd04770b206ae38454134ac is 69, key is db0afee3eab9,42783,1732001853417/rs:state/1732001854081/Put/seqid=0
2024-11-19T07:37:35,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741841_1017 (size=5156)
2024-11-19T07:37:35,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741841_1017 (size=5156)
2024-11-19T07:37:35,137 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4137432ffd04770b206ae38454134ac
2024-11-19T07:37:35,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:35,141 INFO  [RS:0;db0afee3eab9:42783 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:37:35,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42783-0x1015206ead90001, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:35,141 INFO  [RS:0;db0afee3eab9:42783 {}] regionserver.HRegionServer(1031): Exiting; stopping=db0afee3eab9,42783,1732001853417; zookeeper connection closed.
2024-11-19T07:37:35,141 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39e2b786 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39e2b786
2024-11-19T07:37:35,142 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-19T07:37:35,152 DEBUG [M:0;db0afee3eab9:32791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5af12e37dadb4d9ab3acbd51d8cbe386 is 52, key is load_balancer_on/state:d/1732001854756/Put/seqid=0
2024-11-19T07:37:35,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741842_1018 (size=5056)
2024-11-19T07:37:35,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741842_1018 (size=5056)
2024-11-19T07:37:35,156 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5af12e37dadb4d9ab3acbd51d8cbe386
2024-11-19T07:37:35,161 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b6ccc2e7cd24f45ba6b5c443428dff8 as hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b6ccc2e7cd24f45ba6b5c443428dff8
2024-11-19T07:37:35,165 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b6ccc2e7cd24f45ba6b5c443428dff8, entries=8, sequenceid=29, filesize=5.5 K
2024-11-19T07:37:35,166 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977eb86f92f248e5b3b7ca5f08b38d91 as hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/977eb86f92f248e5b3b7ca5f08b38d91
2024-11-19T07:37:35,169 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/977eb86f92f248e5b3b7ca5f08b38d91, entries=3, sequenceid=29, filesize=5.2 K
2024-11-19T07:37:35,170 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e4137432ffd04770b206ae38454134ac as hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4137432ffd04770b206ae38454134ac
2024-11-19T07:37:35,174 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e4137432ffd04770b206ae38454134ac, entries=1, sequenceid=29, filesize=5.0 K
2024-11-19T07:37:35,174 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5af12e37dadb4d9ab3acbd51d8cbe386 as hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5af12e37dadb4d9ab3acbd51d8cbe386
2024-11-19T07:37:35,178 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45495/user/jenkins/test-data/3807236c-36aa-1959-2ff8-1ff74aea993d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5af12e37dadb4d9ab3acbd51d8cbe386, entries=1, sequenceid=29, filesize=4.9 K
2024-11-19T07:37:35,179 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 107ms, sequenceid=29, compaction requested=false
2024-11-19T07:37:35,180 INFO  [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-19T07:37:35,180 DEBUG [M:0;db0afee3eab9:32791 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732001855072Disabling compacts and flushes for region at 1732001855072Disabling writes for close at 1732001855072Obtaining lock to block concurrent updates at 1732001855072Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732001855072Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732001855073 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732001855074 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732001855074Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732001855090 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732001855090Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732001855098 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732001855111 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732001855111Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732001855120 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732001855131 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732001855131Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732001855140 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732001855152 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732001855152Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66ad3413: reopening flushed file at 1732001855160 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a623932: reopening flushed file at 1732001855165 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73f3e027: reopening flushed file at 1732001855169 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bedfe21: reopening flushed file at 1732001855174 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 107ms, sequenceid=29, compaction requested=false at 1732001855179 (+5 ms)Writing region close event to WAL at 1732001855180 (+1 ms)Closed at 1732001855180
2024-11-19T07:37:35,180 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,180 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,180 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,180 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,181 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-19T07:37:35,181 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40659/user/jenkins/test-data/379039ea-b1b5-bdb5-40fa-ec5407a91332/WALs/db0afee3eab9,42239,1732001656169/db0afee3eab9%2C42239%2C1732001656169.meta.1732001657163.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-19T07:37:35,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741830_1006 (size=10311)
2024-11-19T07:37:35,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44629 is added to blk_1073741830_1006 (size=10311)
2024-11-19T07:37:35,183 INFO  [M:0;db0afee3eab9:32791 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-19T07:37:35,183 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-19T07:37:35,183 INFO  [M:0;db0afee3eab9:32791 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32791
2024-11-19T07:37:35,183 INFO  [M:0;db0afee3eab9:32791 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-19T07:37:35,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:35,294 INFO  [M:0;db0afee3eab9:32791 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-19T07:37:35,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32791-0x1015206ead90000, quorum=127.0.0.1:59789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-19T07:37:35,298 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@73bc7eb2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:35,298 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@624ed4c3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:35,298 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:35,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@753778e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:35,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@477d322e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:35,300 WARN  [BP-1830346352-172.17.0.2-1732001851060 heartbeating to localhost/127.0.0.1:45495 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:37:35,300 WARN  [BP-1830346352-172.17.0.2-1732001851060 heartbeating to localhost/127.0.0.1:45495 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1830346352-172.17.0.2-1732001851060 (Datanode Uuid 30644da4-ae01-4ef6-8fa2-267bf45e58c1) service to localhost/127.0.0.1:45495
2024-11-19T07:37:35,301 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:37:35,301 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:37:35,301 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data3/current/BP-1830346352-172.17.0.2-1732001851060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:35,302 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data4/current/BP-1830346352-172.17.0.2-1732001851060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:35,302 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:37:35,305 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@509dedeb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-19T07:37:35,305 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@46761010{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:35,305 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:35,305 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ecbf0f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:35,305 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75728f34{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:35,307 WARN  [BP-1830346352-172.17.0.2-1732001851060 heartbeating to localhost/127.0.0.1:45495 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-19T07:37:35,307 WARN  [BP-1830346352-172.17.0.2-1732001851060 heartbeating to localhost/127.0.0.1:45495 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1830346352-172.17.0.2-1732001851060 (Datanode Uuid 7b3f1de5-3822-451d-81e2-dca5cec8acda) service to localhost/127.0.0.1:45495
2024-11-19T07:37:35,307 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-19T07:37:35,307 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-19T07:37:35,307 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data1/current/BP-1830346352-172.17.0.2-1732001851060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:35,308 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/cluster_8c7f34da-6ab3-62e5-2483-c210ca731a11/data/data2/current/BP-1830346352-172.17.0.2-1732001851060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-19T07:37:35,308 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-19T07:37:35,314 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@71465f4c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-19T07:37:35,315 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2191d18b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-19T07:37:35,315 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-19T07:37:35,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b825638{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-19T07:37:35,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3698cd86{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f8be598d-408a-fb7a-016d-b4fea915df71/hadoop.log.dir/,STOPPED}
2024-11-19T07:37:35,321 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-19T07:37:35,334 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-19T07:37:35,342 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=270 (was 232)
Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/db0afee3eab9:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:45495
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45495 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45495 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45495
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45495 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=536 (was 521) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 147), ProcessCount=11 (was 11), AvailableMemoryMB=12012 (was 12019)