2024-12-02 18:28:25,325 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-02 18:28:25,340 main DEBUG Took 0.012917 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-02 18:28:25,341 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-02 18:28:25,341 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-02 18:28:25,342 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-02 18:28:25,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,351 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-02 18:28:25,362 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,366 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,368 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,369 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,369 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,370 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,370 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,371 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,373 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,374 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,375 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,376 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,377 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-02 18:28:25,378 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,379 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-02 18:28:25,381 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-02 18:28:25,382 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-02 18:28:25,384 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-02 18:28:25,385 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-02 18:28:25,387 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-02 18:28:25,388 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-02 18:28:25,401 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-02 18:28:25,405 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-02 18:28:25,407 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-02 18:28:25,408 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-02 18:28:25,408 main DEBUG createAppenders(={Console})
2024-12-02 18:28:25,409 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-02 18:28:25,410 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-02 18:28:25,410 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-02 18:28:25,411 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-02 18:28:25,411 main DEBUG OutputStream closed
2024-12-02 18:28:25,412 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-02 18:28:25,412 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-02 18:28:25,412 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-02 18:28:25,482 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-02 18:28:25,484 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-02 18:28:25,485 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-02 18:28:25,486 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-02 18:28:25,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-02 18:28:25,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-02 18:28:25,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-02 18:28:25,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-02 18:28:25,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-02 18:28:25,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-02 18:28:25,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-02 18:28:25,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-02 18:28:25,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-02 18:28:25,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-02 18:28:25,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-02 18:28:25,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-02 18:28:25,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-02 18:28:25,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-02 18:28:25,495 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-02 18:28:25,495 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-02 18:28:25,496 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-02 18:28:25,497 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-02T18:28:25,743 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a
2024-12-02 18:28:25,746 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-02 18:28:25,747 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-02T18:28:25,755 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-02T18:28:25,792 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=4875
2024-12-02T18:28:25,795 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:28:25,808 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309, deleteOnExit=true
2024-12-02T18:28:25,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:28:25,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/test.cache.data in system properties and HBase conf
2024-12-02T18:28:25,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:28:25,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:28:25,811 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:28:25,811 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:28:25,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:28:25,899 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-02T18:28:26,008 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:28:26,012 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:28:26,013 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:28:26,014 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:28:26,014 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:28:26,015 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:28:26,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:28:26,016 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:28:26,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:28:26,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:28:26,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:28:26,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:28:26,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:28:26,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:28:26,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:28:26,425 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:28:27,173 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-02T18:28:27,237 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2622ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-02T18:28:27,298 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:28:27,358 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:28:27,377 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:28:27,377 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:28:27,378 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:28:27,389 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:28:27,392 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:28:27,393 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:28:27,570 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/java.io.tmpdir/jetty-localhost-40607-hadoop-hdfs-3_4_1-tests_jar-_-any-13081811146202304719/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:28:27,578 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:40607}
2024-12-02T18:28:27,578 INFO  [Time-limited test {}] server.Server(415): Started @2964ms
2024-12-02T18:28:27,610 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:28:28,148 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:28:28,154 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:28:28,155 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:28:28,155 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:28:28,156 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:28:28,156 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:28:28,157 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:28:28,258 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/java.io.tmpdir/jetty-localhost-39521-hadoop-hdfs-3_4_1-tests_jar-_-any-10402495896420988210/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:28:28,259 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:39521}
2024-12-02T18:28:28,259 INFO  [Time-limited test {}] server.Server(415): Started @3645ms
2024-12-02T18:28:28,310 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:28:28,410 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:28:28,416 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:28:28,418 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:28:28,419 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:28:28,419 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:28:28,420 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:28:28,421 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:28:28,546 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/java.io.tmpdir/jetty-localhost-40973-hadoop-hdfs-3_4_1-tests_jar-_-any-17585621243345732914/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:28:28,547 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:40973}
2024-12-02T18:28:28,547 INFO  [Time-limited test {}] server.Server(415): Started @3933ms
2024-12-02T18:28:28,550 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:28:30,209 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data1/current/BP-1810398651-172.17.0.3-1733164106500/current, will proceed with Du for space computation calculation, 
2024-12-02T18:28:30,209 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data2/current/BP-1810398651-172.17.0.3-1733164106500/current, will proceed with Du for space computation calculation, 
2024-12-02T18:28:30,225 WARN  [Thread-103 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data4/current/BP-1810398651-172.17.0.3-1733164106500/current, will proceed with Du for space computation calculation, 
2024-12-02T18:28:30,225 WARN  [Thread-102 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data3/current/BP-1810398651-172.17.0.3-1733164106500/current, will proceed with Du for space computation calculation, 
2024-12-02T18:28:30,244 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:28:30,247 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:28:30,291 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x374546fd1fdfcc39 with lease ID 0x1c5ed3079ade129a: Processing first storage report for DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba from datanode DatanodeRegistration(127.0.0.1:36031, datanodeUuid=f4926761-ad1f-4d40-b334-fc1ca3f97dc8, infoPort=35241, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500)
2024-12-02T18:28:30,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x374546fd1fdfcc39 with lease ID 0x1c5ed3079ade129a: from storage DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba node DatanodeRegistration(127.0.0.1:36031, datanodeUuid=f4926761-ad1f-4d40-b334-fc1ca3f97dc8, infoPort=35241, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-02T18:28:30,292 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82301cf045da7cb4 with lease ID 0x1c5ed3079ade129b: Processing first storage report for DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701 from datanode DatanodeRegistration(127.0.0.1:38263, datanodeUuid=e4c71843-62f8-47ee-8357-96edf56da42e, infoPort=34733, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500)
2024-12-02T18:28:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82301cf045da7cb4 with lease ID 0x1c5ed3079ade129b: from storage DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701 node DatanodeRegistration(127.0.0.1:38263, datanodeUuid=e4c71843-62f8-47ee-8357-96edf56da42e, infoPort=34733, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:28:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x374546fd1fdfcc39 with lease ID 0x1c5ed3079ade129a: Processing first storage report for DS-2f78da29-ff14-469b-a0ab-c63359521586 from datanode DatanodeRegistration(127.0.0.1:36031, datanodeUuid=f4926761-ad1f-4d40-b334-fc1ca3f97dc8, infoPort=35241, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500)
2024-12-02T18:28:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x374546fd1fdfcc39 with lease ID 0x1c5ed3079ade129a: from storage DS-2f78da29-ff14-469b-a0ab-c63359521586 node DatanodeRegistration(127.0.0.1:36031, datanodeUuid=f4926761-ad1f-4d40-b334-fc1ca3f97dc8, infoPort=35241, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:28:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82301cf045da7cb4 with lease ID 0x1c5ed3079ade129b: Processing first storage report for DS-3d803cd5-d894-4ef0-98e7-3e43614114a3 from datanode DatanodeRegistration(127.0.0.1:38263, datanodeUuid=e4c71843-62f8-47ee-8357-96edf56da42e, infoPort=34733, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500)
2024-12-02T18:28:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82301cf045da7cb4 with lease ID 0x1c5ed3079ade129b: from storage DS-3d803cd5-d894-4ef0-98e7-3e43614114a3 node DatanodeRegistration(127.0.0.1:38263, datanodeUuid=e4c71843-62f8-47ee-8357-96edf56da42e, infoPort=34733, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=872534856;c=1733164106500), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:28:30,368 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a
2024-12-02T18:28:30,432 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/zookeeper_0, clientPort=50294, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:28:30,440 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50294
2024-12-02T18:28:30,450 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:30,454 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:30,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:28:30,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:28:31,053 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25 with version=8
2024-12-02T18:28:31,053 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:28:31,139 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-02T18:28:31,365 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:28:31,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:31,376 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:31,383 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:28:31,383 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:31,384 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:28:31,532 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:28:31,595 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-02T18:28:31,603 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-02T18:28:31,607 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:28:31,628 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 73736 (auto-detected)
2024-12-02T18:28:31,629 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-12-02T18:28:31,646 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43055
2024-12-02T18:28:31,665 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43055 connecting to ZooKeeper ensemble=127.0.0.1:50294
2024-12-02T18:28:31,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430550x0, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:28:31,774 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43055-0x101974d8dfd0000 connected
2024-12-02T18:28:31,867 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:31,874 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:31,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:28:31,889 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25, hbase.cluster.distributed=false
2024-12-02T18:28:31,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:28:31,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43055
2024-12-02T18:28:31,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43055
2024-12-02T18:28:31,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43055
2024-12-02T18:28:31,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43055
2024-12-02T18:28:31,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43055
2024-12-02T18:28:32,013 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:28:32,015 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:32,015 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:32,015 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:28:32,015 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:28:32,016 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:28:32,018 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:28:32,021 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:28:32,021 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46367
2024-12-02T18:28:32,023 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46367 connecting to ZooKeeper ensemble=127.0.0.1:50294
2024-12-02T18:28:32,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:32,026 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:32,042 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463670x0, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:28:32,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463670x0, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:28:32,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46367-0x101974d8dfd0001 connected
2024-12-02T18:28:32,048 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:28:32,057 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:28:32,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:28:32,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:28:32,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46367
2024-12-02T18:28:32,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46367
2024-12-02T18:28:32,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46367
2024-12-02T18:28:32,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46367
2024-12-02T18:28:32,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46367
2024-12-02T18:28:32,079 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:43055
2024-12-02T18:28:32,080 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,096 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:28:32,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:28:32,098 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,126 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:28:32,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,127 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,128 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:28:32,129 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,43055,1733164111212 from backup master directory
2024-12-02T18:28:32,137 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:28:32,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:28:32,138 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:28:32,138 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,140 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-02T18:28:32,142 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-02T18:28:32,193 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase.id] with ID: 73d62189-8c05-4cf7-9072-4a8272d8f1a8
2024-12-02T18:28:32,194 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/.tmp/hbase.id
2024-12-02T18:28:32,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:28:32,211 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:28:32,212 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/.tmp/hbase.id]:[hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase.id]
2024-12-02T18:28:32,259 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:32,265 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:28:32,281 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms.
2024-12-02T18:28:32,295 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:28:32,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:28:32,331 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:28:32,332 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:28:32,336 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:28:32,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:28:32,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:28:32,382 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store
2024-12-02T18:28:32,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:28:32,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:28:32,407 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-02T18:28:32,411 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:32,413 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:28:32,413 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:28:32,413 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:28:32,415 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:28:32,415 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:28:32,415 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:28:32,416 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164112412Disabling compacts and flushes for region at 1733164112412Disabling writes for close at 1733164112415 (+3 ms)Writing region close event to WAL at 1733164112415Closed at 1733164112415
2024-12-02T18:28:32,418 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/.initializing
2024-12-02T18:28:32,418 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/WALs/95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,440 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C43055%2C1733164111212, suffix=, logDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/WALs/95451c90bed2,43055,1733164111212, archiveDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/oldWALs, maxLogs=10
2024-12-02T18:28:32,448 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43055%2C1733164111212.1733164112444
2024-12-02T18:28:32,467 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/WALs/95451c90bed2,43055,1733164111212/95451c90bed2%2C43055%2C1733164111212.1733164112444
2024-12-02T18:28:32,474 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35241:35241),(127.0.0.1/127.0.0.1:34733:34733)]
2024-12-02T18:28:32,475 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:28:32,476 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:32,478 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,479 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,512 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:28:32,532 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:28:32,535 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:32,537 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:32,538 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:28:32,541 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:28:32,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:32,542 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:28:32,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:28:32,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:28:32,546 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:32,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:28:32,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:28:32,550 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:28:32,550 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:32,551 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:28:32,551 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,555 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,556 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,561 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,562 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,565 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:28:32,568 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:28:32,572 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:28:32,573 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877203, jitterRate=0.11542162299156189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:28:32,579 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164112491Initializing all the Stores at 1733164112493 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164112493Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164112494 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164112494Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164112494Cleaning up temporary data from old regions at 1733164112562 (+68 ms)Region opened successfully at 1733164112579 (+17 ms)
2024-12-02T18:28:32,580 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:28:32,610 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7833035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:28:32,636 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:28:32,646 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:28:32,646 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:28:32,649 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:28:32,650 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-02T18:28:32,654 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec
2024-12-02T18:28:32,654 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:28:32,677 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:28:32,688 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:28:32,786 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:28:32,793 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:28:32,795 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:28:32,810 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:28:32,813 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:28:32,818 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:28:32,829 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:28:32,830 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:28:32,842 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:28:32,859 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:28:32,871 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:28:32,884 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:28:32,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:28:32,885 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,889 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,43055,1733164111212, sessionid=0x101974d8dfd0000, setting cluster-up flag (Was=false)
2024-12-02T18:28:32,924 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,958 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:28:32,961 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,43055,1733164111212
2024-12-02T18:28:32,990 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:32,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:33,021 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:28:33,024 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,43055,1733164111212
2024-12-02T18:28:33,030 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:28:33,070 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(746): ClusterId : 73d62189-8c05-4cf7-9072-4a8272d8f1a8
2024-12-02T18:28:33,072 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:28:33,086 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:28:33,087 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:28:33,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:28:33,096 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:28:33,097 DEBUG [RS:0;95451c90bed2:46367 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6d68d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:28:33,103 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:28:33,109 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:28:33,110 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:46367
2024-12-02T18:28:33,113 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:28:33,113 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:28:33,113 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:28:33,115 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,43055,1733164111212 with port=46367, startcode=1733164111982
2024-12-02T18:28:33,114 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,43055,1733164111212
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:28:33,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,121 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:28:33,121 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,122 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164143122
2024-12-02T18:28:33,124 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:28:33,125 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:28:33,125 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:28:33,126 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:28:33,127 DEBUG [RS:0;95451c90bed2:46367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:28:33,129 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:28:33,130 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:28:33,130 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:28:33,131 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:28:33,131 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:33,131 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:28:33,131 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,135 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:28:33,136 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:28:33,137 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:28:33,140 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:28:33,140 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:28:33,145 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164113142,5,FailOnTimeoutGroup]
2024-12-02T18:28:33,147 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164113145,5,FailOnTimeoutGroup]
2024-12-02T18:28:33,147 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:28:33,147 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:28:33,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:28:33,148 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,149 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,150 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:28:33,151 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25
2024-12-02T18:28:33,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:28:33,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:28:33,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:33,168 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:28:33,172 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:28:33,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:33,174 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:33,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:28:33,179 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:28:33,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:33,181 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:33,181 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:28:33,185 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:28:33,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:33,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:33,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:28:33,189 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:28:33,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:33,191 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:33,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:28:33,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740
2024-12-02T18:28:33,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740
2024-12-02T18:28:33,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:28:33,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:28:33,197 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:28:33,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:28:33,203 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57811, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:28:33,204 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:28:33,206 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812509, jitterRate=0.03315918147563934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:28:33,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164113164Initializing all the Stores at 1733164113166 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164113166Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164113168 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164113168Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164113168Cleaning up temporary data from old regions at 1733164113196 (+28 ms)Region opened successfully at 1733164113209 (+13 ms)
2024-12-02T18:28:33,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:28:33,210 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:28:33,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:28:33,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:28:33,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:28:33,211 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:28:33,211 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43055 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164113209Disabling compacts and flushes for region at 1733164113209Disabling writes for close at 1733164113210 (+1 ms)Writing region close event to WAL at 1733164113211 (+1 ms)Closed at 1733164113211
2024-12-02T18:28:33,214 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43055 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,215 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:28:33,215 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:28:33,224 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:28:33,227 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25
2024-12-02T18:28:33,227 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36351
2024-12-02T18:28:33,227 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:28:33,232 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:28:33,235 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:28:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:28:33,240 DEBUG [RS:0;95451c90bed2:46367 {}] zookeeper.ZKUtil(111): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,240 WARN  [RS:0;95451c90bed2:46367 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:28:33,240 INFO  [RS:0;95451c90bed2:46367 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:28:33,240 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,242 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,46367,1733164111982]
2024-12-02T18:28:33,265 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:28:33,278 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:28:33,282 INFO  [RS:0;95451c90bed2:46367 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:28:33,283 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,283 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:28:33,288 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:28:33,289 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,290 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,290 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,290 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,290 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,290 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,291 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,292 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:28:33,292 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:28:33,292 DEBUG [RS:0;95451c90bed2:46367 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:28:33,293 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,293 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,293 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,293 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,293 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,294 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46367,1733164111982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:28:33,309 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:28:33,311 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46367,1733164111982-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,311 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,311 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.Replication(171): 95451c90bed2,46367,1733164111982 started
2024-12-02T18:28:33,326 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:33,326 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,46367,1733164111982, RpcServer on 95451c90bed2/172.17.0.3:46367, sessionid=0x101974d8dfd0001
2024-12-02T18:28:33,327 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:28:33,327 DEBUG [RS:0;95451c90bed2:46367 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,327 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,46367,1733164111982'
2024-12-02T18:28:33,327 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:28:33,328 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:28:33,329 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:28:33,329 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:28:33,329 DEBUG [RS:0;95451c90bed2:46367 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,329 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,46367,1733164111982'
2024-12-02T18:28:33,329 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:28:33,330 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:28:33,331 DEBUG [RS:0;95451c90bed2:46367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:28:33,331 INFO  [RS:0;95451c90bed2:46367 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:28:33,331 INFO  [RS:0;95451c90bed2:46367 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:28:33,386 WARN  [95451c90bed2:43055 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:28:33,445 INFO  [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C46367%2C1733164111982, suffix=, logDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982, archiveDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs, maxLogs=32
2024-12-02T18:28:33,447 INFO  [RS:0;95451c90bed2:46367 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164113447
2024-12-02T18:28:33,455 INFO  [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164113447
2024-12-02T18:28:33,456 DEBUG [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34733:34733),(127.0.0.1/127.0.0.1:35241:35241)]
2024-12-02T18:28:33,638 DEBUG [95451c90bed2:43055 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:28:33,648 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,46367,1733164111982
2024-12-02T18:28:33,654 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,46367,1733164111982, state=OPENING
2024-12-02T18:28:33,882 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:28:33,895 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:33,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:28:33,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:28:33,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:28:33,899 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:28:33,902 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,46367,1733164111982}]
2024-12-02T18:28:34,077 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:28:34,080 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51825, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:28:34,092 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:28:34,093 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:28:34,096 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C46367%2C1733164111982.meta, suffix=.meta, logDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982, archiveDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs, maxLogs=32
2024-12-02T18:28:34,098 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.meta.1733164114098.meta
2024-12-02T18:28:34,106 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.meta.1733164114098.meta
2024-12-02T18:28:34,108 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35241:35241),(127.0.0.1/127.0.0.1:34733:34733)]
2024-12-02T18:28:34,109 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:28:34,111 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:28:34,113 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:28:34,119 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:28:34,125 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:28:34,126 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:34,127 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:28:34,127 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:28:34,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:28:34,133 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:28:34,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:34,134 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:34,135 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:28:34,137 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:28:34,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:34,138 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:34,138 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:28:34,140 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:28:34,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:34,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:34,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:28:34,144 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:28:34,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:34,145 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:28:34,145 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:28:34,147 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740
2024-12-02T18:28:34,151 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740
2024-12-02T18:28:34,154 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:28:34,155 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:28:34,156 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:28:34,159 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:28:34,160 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804365, jitterRate=0.02280372381210327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:28:34,161 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:28:34,162 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164114128Writing region info on filesystem at 1733164114128Initializing all the Stores at 1733164114130 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164114130Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164114130Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164114130Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164114130Cleaning up temporary data from old regions at 1733164114155 (+25 ms)Running coprocessor post-open hooks at 1733164114161 (+6 ms)Region opened successfully at 1733164114161
2024-12-02T18:28:34,168 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164114069
2024-12-02T18:28:34,180 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:28:34,181 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:28:34,182 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,46367,1733164111982
2024-12-02T18:28:34,184 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,46367,1733164111982, state=OPEN
2024-12-02T18:28:34,228 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:28:34,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:28:34,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:28:34,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:28:34,228 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,46367,1733164111982
2024-12-02T18:28:34,234 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:28:34,234 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,46367,1733164111982 in 326 msec
2024-12-02T18:28:34,242 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:28:34,242 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0130 sec
2024-12-02T18:28:34,244 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:28:34,244 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:28:34,263 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:28:34,264 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,46367,1733164111982, seqNum=-1]
2024-12-02T18:28:34,292 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:28:34,295 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45727, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:28:34,316 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2590 sec
2024-12-02T18:28:34,317 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164114316, completionTime=-1
2024-12-02T18:28:34,319 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:28:34,320 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:28:34,354 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:28:34,354 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164174354
2024-12-02T18:28:34,354 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164234354
2024-12-02T18:28:34,354 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec
2024-12-02T18:28:34,358 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,358 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,358 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,360 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:43055, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,360 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,361 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,368 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:28:34,393 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.255sec
2024-12-02T18:28:34,394 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:28:34,396 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:28:34,397 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:28:34,398 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:28:34,398 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:28:34,399 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:28:34,400 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:28:34,409 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:28:34,410 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:28:34,410 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43055,1733164111212-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:28:34,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358a6cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:28:34,481 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-02T18:28:34,482 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-02T18:28:34,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,43055,-1 for getting cluster id
2024-12-02T18:28:34,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:28:34,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '73d62189-8c05-4cf7-9072-4a8272d8f1a8'
2024-12-02T18:28:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:28:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "73d62189-8c05-4cf7-9072-4a8272d8f1a8"

2024-12-02T18:28:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2461ffc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:28:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,43055,-1]
2024-12-02T18:28:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:28:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:28:34,507 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:28:34,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f64e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:28:34,511 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:28:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,46367,1733164111982, seqNum=-1]
2024-12-02T18:28:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:28:34,521 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:28:34,556 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,43055,1733164111212
2024-12-02T18:28:34,557 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:28:34,563 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:28:34,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-02T18:28:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 95451c90bed2,43055,1733164111212
2024-12-02T18:28:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c3c746e
2024-12-02T18:28:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-02T18:28:34,581 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-02T18:28:34,583 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-02T18:28:34,583 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-02T18:28:34,588 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:28:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-02T18:28:34,600 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-02T18:28:34,602 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-02T18:28:34,602 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:34,605 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-02T18:28:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:28:34,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741835_1011 (size=389)
2024-12-02T18:28:34,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741835_1011 (size=389)
2024-12-02T18:28:34,643 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c466fedace16710fc1f63981c747dae9, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25
2024-12-02T18:28:34,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741836_1012 (size=72)
2024-12-02T18:28:34,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741836_1012 (size=72)
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing c466fedace16710fc1f63981c747dae9, disabling compactions & flushes
2024-12-02T18:28:34,653 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9. after waiting 0 ms
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:34,653 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:34,653 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for c466fedace16710fc1f63981c747dae9:
Waiting for close lock at 1733164114653Disabling compacts and flushes for region at 1733164114653Disabling writes for close at 1733164114653Writing region close event to WAL at 1733164114653Closed at 1733164114653
2024-12-02T18:28:34,655 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-02T18:28:34,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733164114655"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164114655"}]},"ts":"1733164114655"}
2024-12-02T18:28:34,664 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-02T18:28:34,666 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-02T18:28:34,668 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164114666"}]},"ts":"1733164114666"}
2024-12-02T18:28:34,673 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-02T18:28:34,675 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c466fedace16710fc1f63981c747dae9, ASSIGN}]
2024-12-02T18:28:34,678 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c466fedace16710fc1f63981c747dae9, ASSIGN
2024-12-02T18:28:34,680 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c466fedace16710fc1f63981c747dae9, ASSIGN; state=OFFLINE, location=95451c90bed2,46367,1733164111982; forceNewPlan=false, retain=false
2024-12-02T18:28:34,832 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c466fedace16710fc1f63981c747dae9, regionState=OPENING, regionLocation=95451c90bed2,46367,1733164111982
2024-12-02T18:28:34,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c466fedace16710fc1f63981c747dae9, ASSIGN because future has completed
2024-12-02T18:28:34,841 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c466fedace16710fc1f63981c747dae9, server=95451c90bed2,46367,1733164111982}]
2024-12-02T18:28:35,008 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:35,008 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c466fedace16710fc1f63981c747dae9, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:28:35,009 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,009 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:28:35,009 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,009 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,012 INFO  [StoreOpener-c466fedace16710fc1f63981c747dae9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c466fedace16710fc1f63981c747dae9 
2024-12-02T18:28:35,014 INFO  [StoreOpener-c466fedace16710fc1f63981c747dae9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c466fedace16710fc1f63981c747dae9 columnFamilyName info
2024-12-02T18:28:35,015 DEBUG [StoreOpener-c466fedace16710fc1f63981c747dae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:28:35,016 INFO  [StoreOpener-c466fedace16710fc1f63981c747dae9-1 {}] regionserver.HStore(327): Store=c466fedace16710fc1f63981c747dae9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:28:35,016 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,018 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,018 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,019 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,019 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,022 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,025 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:28:35,026 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c466fedace16710fc1f63981c747dae9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724121, jitterRate=-0.07923287153244019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:28:35,026 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:35,027 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c466fedace16710fc1f63981c747dae9:
Running coprocessor pre-open hook at 1733164115009Writing region info on filesystem at 1733164115009Initializing all the Stores at 1733164115011 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164115011Cleaning up temporary data from old regions at 1733164115019 (+8 ms)Running coprocessor post-open hooks at 1733164115026 (+7 ms)Region opened successfully at 1733164115027 (+1 ms)
2024-12-02T18:28:35,029 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9., pid=6, masterSystemTime=1733164114996
2024-12-02T18:28:35,033 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:35,033 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:35,034 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c466fedace16710fc1f63981c747dae9, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,46367,1733164111982
2024-12-02T18:28:35,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c466fedace16710fc1f63981c747dae9, server=95451c90bed2,46367,1733164111982 because future has completed
2024-12-02T18:28:35,046 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-02T18:28:35,046 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c466fedace16710fc1f63981c747dae9, server=95451c90bed2,46367,1733164111982 in 201 msec
2024-12-02T18:28:35,052 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-02T18:28:35,052 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c466fedace16710fc1f63981c747dae9, ASSIGN in 371 msec
2024-12-02T18:28:35,054 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-02T18:28:35,054 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164115054"}]},"ts":"1733164115054"}
2024-12-02T18:28:35,058 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-02T18:28:35,060 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-02T18:28:35,063 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 468 msec
2024-12-02T18:28:39,642 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-02T18:28:39,695 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:28:39,696 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-02T18:28:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:28:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-02T18:28:41,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-02T18:28:41,594 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-02T18:28:41,594 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:28:41,594 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-02T18:28:41,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:28:41,595 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-02T18:28:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:28:44,707 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-02T18:28:44,709 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-02T18:28:44,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-02T18:28:44,717 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:28:44,717 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164124717
2024-12-02T18:28:44,727 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:28:44,727 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:28:44,727 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:28:44,728 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:28:44,728 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:28:44,728 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164113447 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164124717
2024-12-02T18:28:44,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34733:34733),(127.0.0.1/127.0.0.1:35241:35241)]
2024-12-02T18:28:44,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164113447 is not closed yet, will try archiving it next time
2024-12-02T18:28:44,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741833_1009 (size=451)
2024-12-02T18:28:44,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741833_1009 (size=451)
2024-12-02T18:28:44,734 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164113447 to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs/95451c90bed2%2C46367%2C1733164111982.1733164113447
2024-12-02T18:28:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9., hostname=95451c90bed2,46367,1733164111982, seqNum=2]
2024-12-02T18:28:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46367 {}] regionserver.HRegion(8855): Flush requested on c466fedace16710fc1f63981c747dae9
2024-12-02T18:28:56,789 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c466fedace16710fc1f63981c747dae9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:28:56,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/8c679b6dcf654667ae1ff86965650f70 is 1080, key is row0001/info:/1733164124742/Put/seqid=0
2024-12-02T18:28:56,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741838_1014 (size=12509)
2024-12-02T18:28:56,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741838_1014 (size=12509)
2024-12-02T18:28:56,879 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/8c679b6dcf654667ae1ff86965650f70
2024-12-02T18:28:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/8c679b6dcf654667ae1ff86965650f70 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70
2024-12-02T18:28:56,953 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70, entries=7, sequenceid=11, filesize=12.2 K
2024-12-02T18:28:56,965 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c466fedace16710fc1f63981c747dae9 in 173ms, sequenceid=11, compaction requested=false
2024-12-02T18:28:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c466fedace16710fc1f63981c747dae9:

2024-12-02T18:29:00,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:29:04,799 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164144798
2024-12-02T18:29:05,010 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:05,010 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:05,010 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:05,010 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:05,011 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:05,011 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:05,011 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164124717 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164144798
2024-12-02T18:29:05,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741837_1013 (size=12399)
2024-12-02T18:29:05,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741837_1013 (size=12399)
2024-12-02T18:29:05,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35241:35241),(127.0.0.1/127.0.0.1:34733:34733)]
2024-12-02T18:29:05,222 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:07,426 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:09,630 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:11,833 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46367 {}] regionserver.HRegion(8855): Flush requested on c466fedace16710fc1f63981c747dae9
2024-12-02T18:29:11,834 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c466fedace16710fc1f63981c747dae9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:29:12,035 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:12,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/1a574747bc784a049b127440eb35ef18 is 1080, key is row0008/info:/1733164138787/Put/seqid=0
2024-12-02T18:29:12,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741840_1016 (size=12509)
2024-12-02T18:29:12,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741840_1016 (size=12509)
2024-12-02T18:29:12,050 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/1a574747bc784a049b127440eb35ef18
2024-12-02T18:29:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/1a574747bc784a049b127440eb35ef18 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18
2024-12-02T18:29:12,076 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18, entries=7, sequenceid=21, filesize=12.2 K
2024-12-02T18:29:12,278 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:12,278 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c466fedace16710fc1f63981c747dae9 in 445ms, sequenceid=21, compaction requested=false
2024-12-02T18:29:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c466fedace16710fc1f63981c747dae9:

2024-12-02T18:29:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-02T18:29:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70 because midkey is the same as first or last row
2024-12-02T18:29:14,039 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:14,424 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-02T18:29:14,424 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-02T18:29:16,245 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:16,252 WARN  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:16,253 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C46367%2C1733164111982:(num 1733164144798) roll requested
2024-12-02T18:29:16,254 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164156253
2024-12-02T18:29:16,466 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 209 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:16,466 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:16,466 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:16,466 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:16,466 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:16,466 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:16,467 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164144798 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164156253
2024-12-02T18:29:16,468 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35241:35241),(127.0.0.1/127.0.0.1:34733:34733)]
2024-12-02T18:29:16,468 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164144798 is not closed yet, will try archiving it next time
2024-12-02T18:29:16,468 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164124717 to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs/95451c90bed2%2C46367%2C1733164111982.1733164124717
2024-12-02T18:29:16,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741839_1015 (size=7739)
2024-12-02T18:29:16,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741839_1015 (size=7739)
2024-12-02T18:29:18,456 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:20,009 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region c466fedace16710fc1f63981c747dae9, had cached 0 bytes from a total of 25018
2024-12-02T18:29:20,661 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:22,867 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:25,072 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:27,076 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-02T18:29:27,077 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164167077
2024-12-02T18:29:30,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:29:32,094 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:32,096 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK]]
2024-12-02T18:29:32,097 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C46367%2C1733164111982:(num 1733164167077) roll requested
2024-12-02T18:29:32,097 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:32,097 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:32,097 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:32,097 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:32,097 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:32,098 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164156253 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164167077
2024-12-02T18:29:32,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34733:34733),(127.0.0.1/127.0.0.1:35241:35241)]
2024-12-02T18:29:32,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164156253 is not closed yet, will try archiving it next time
2024-12-02T18:29:32,099 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164172099
2024-12-02T18:29:32,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741841_1017 (size=4753)
2024-12-02T18:29:32,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741841_1017 (size=4753)
2024-12-02T18:29:37,106 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:37,106 WARN  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46367 {}] regionserver.HRegion(8855): Flush requested on c466fedace16710fc1f63981c747dae9
2024-12-02T18:29:37,108 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c466fedace16710fc1f63981c747dae9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:29:37,121 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5018 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:37,121 WARN  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5018 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:39,109 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-02T18:29:42,110 INFO  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:42,110 WARN  [FSHLog-0-hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25-prefix:95451c90bed2,46367,1733164111982 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38263,DS-8dfcbeba-52f2-4d78-b46d-3a34956fa701,DISK], DatanodeInfoWithStorage[127.0.0.1:36031,DS-9d9b9465-fcc6-4dd4-9893-262bfbccf7ba,DISK]]
2024-12-02T18:29:42,111 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,111 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,111 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,111 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,112 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,112 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164167077 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164172099
2024-12-02T18:29:42,114 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35241:35241),(127.0.0.1/127.0.0.1:34733:34733)]
2024-12-02T18:29:42,114 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164167077 is not closed yet, will try archiving it next time
2024-12-02T18:29:42,114 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C46367%2C1733164111982:(num 1733164182114) roll requested
2024-12-02T18:29:42,114 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164182114
2024-12-02T18:29:42,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741842_1018 (size=1569)
2024-12-02T18:29:42,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741842_1018 (size=1569)
2024-12-02T18:29:42,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/602ba4434bb84627b45d1b191c61cfad is 1080, key is row0015/info:/1733164153837/Put/seqid=0
2024-12-02T18:29:42,126 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,126 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,127 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741845_1021 (size=12509)
2024-12-02T18:29:42,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741845_1021 (size=12509)
2024-12-02T18:29:42,132 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,132 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,132 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164172099 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164182114
2024-12-02T18:29:42,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34733:34733),(127.0.0.1/127.0.0.1:35241:35241)]
2024-12-02T18:29:42,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164172099 is not closed yet, will try archiving it next time
2024-12-02T18:29:42,134 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46367%2C1733164111982.1733164182133
2024-12-02T18:29:42,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741843_1019 (size=93)
2024-12-02T18:29:42,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741843_1019 (size=93)
2024-12-02T18:29:42,136 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164172099 to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs/95451c90bed2%2C46367%2C1733164111982.1733164172099
2024-12-02T18:29:42,141 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,141 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,141 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,141 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,141 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:29:42,141 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164182114 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164182133
2024-12-02T18:29:42,143 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34733:34733),(127.0.0.1/127.0.0.1:35241:35241)]
2024-12-02T18:29:42,143 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/WALs/95451c90bed2,46367,1733164111982/95451c90bed2%2C46367%2C1733164111982.1733164182114 is not closed yet, will try archiving it next time
2024-12-02T18:29:42,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741844_1020 (size=1258)
2024-12-02T18:29:42,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741844_1020 (size=1258)
2024-12-02T18:29:42,531 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/602ba4434bb84627b45d1b191c61cfad
2024-12-02T18:29:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/602ba4434bb84627b45d1b191c61cfad as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad
2024-12-02T18:29:42,600 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad, entries=7, sequenceid=31, filesize=12.2 K
2024-12-02T18:29:42,602 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=1.05 KB/1076 for c466fedace16710fc1f63981c747dae9 in 5494ms, sequenceid=31, compaction requested=true
2024-12-02T18:29:42,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c466fedace16710fc1f63981c747dae9:

2024-12-02T18:29:42,602 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-02T18:29:42,602 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:42,602 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70 because midkey is the same as first or last row
2024-12-02T18:29:42,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c466fedace16710fc1f63981c747dae9:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:29:42,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:29:42,605 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:29:42,608 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:29:42,609 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HStore(1541): c466fedace16710fc1f63981c747dae9/info is initiating minor compaction (all files)
2024-12-02T18:29:42,609 INFO  [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c466fedace16710fc1f63981c747dae9/info in TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:29:42,610 INFO  [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad] into tmpdir=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp, totalSize=36.6 K
2024-12-02T18:29:42,611 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c679b6dcf654667ae1ff86965650f70, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733164124742
2024-12-02T18:29:42,612 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a574747bc784a049b127440eb35ef18, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733164138787
2024-12-02T18:29:42,612 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602ba4434bb84627b45d1b191c61cfad, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733164153837
2024-12-02T18:29:42,639 INFO  [RS:0;95451c90bed2:46367-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c466fedace16710fc1f63981c747dae9#info#compaction#3 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:29:42,640 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/197f3d4382704d00ace55b3875389a06 is 1080, key is row0001/info:/1733164124742/Put/seqid=0
2024-12-02T18:29:42,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741847_1023 (size=27710)
2024-12-02T18:29:42,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741847_1023 (size=27710)
2024-12-02T18:29:42,656 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/197f3d4382704d00ace55b3875389a06 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/197f3d4382704d00ace55b3875389a06
2024-12-02T18:29:42,675 INFO  [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c466fedace16710fc1f63981c747dae9/info of c466fedace16710fc1f63981c747dae9 into 197f3d4382704d00ace55b3875389a06(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:29:42,675 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c466fedace16710fc1f63981c747dae9:

2024-12-02T18:29:42,677 INFO  [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9., storeName=c466fedace16710fc1f63981c747dae9/info, priority=13, startTime=1733164182603; duration=0sec
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/197f3d4382704d00ace55b3875389a06 because midkey is the same as first or last row
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:42,678 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/197f3d4382704d00ace55b3875389a06 because midkey is the same as first or last row
2024-12-02T18:29:42,679 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-02T18:29:42,679 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:42,679 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/197f3d4382704d00ace55b3875389a06 because midkey is the same as first or last row
2024-12-02T18:29:42,679 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:29:42,679 DEBUG [RS:0;95451c90bed2:46367-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c466fedace16710fc1f63981c747dae9:info
2024-12-02T18:29:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46367 {}] regionserver.HRegion(8855): Flush requested on c466fedace16710fc1f63981c747dae9
2024-12-02T18:29:54,165 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c466fedace16710fc1f63981c747dae9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:29:54,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/49f8979568f6420486e887718034b7ec is 1080, key is row0022/info:/1733164182135/Put/seqid=0
2024-12-02T18:29:54,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741848_1024 (size=12509)
2024-12-02T18:29:54,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741848_1024 (size=12509)
2024-12-02T18:29:54,179 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/49f8979568f6420486e887718034b7ec
2024-12-02T18:29:54,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/49f8979568f6420486e887718034b7ec as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/49f8979568f6420486e887718034b7ec
2024-12-02T18:29:54,196 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/49f8979568f6420486e887718034b7ec, entries=7, sequenceid=42, filesize=12.2 K
2024-12-02T18:29:54,198 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c466fedace16710fc1f63981c747dae9 in 33ms, sequenceid=42, compaction requested=false
2024-12-02T18:29:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c466fedace16710fc1f63981c747dae9:

2024-12-02T18:29:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-02T18:29:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:29:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/197f3d4382704d00ace55b3875389a06 because midkey is the same as first or last row
2024-12-02T18:30:00,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:30:02,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:30:02,184 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:02,184 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:02,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:02,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:02,189 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:30:02,189 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:30:02,189 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1575943754, stopped=false
2024-12-02T18:30:02,189 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,43055,1733164111212
2024-12-02T18:30:02,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:02,275 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:02,275 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:02,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:02,275 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:02,275 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:02,275 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:02,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:02,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:02,276 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,46367,1733164111982' *****
2024-12-02T18:30:02,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:02,276 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:30:02,276 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:30:02,276 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:30:02,276 INFO  [RS:0;95451c90bed2:46367 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:30:02,276 INFO  [RS:0;95451c90bed2:46367 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:30:02,277 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(3091): Received CLOSE for c466fedace16710fc1f63981c747dae9
2024-12-02T18:30:02,277 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,46367,1733164111982
2024-12-02T18:30:02,277 INFO  [RS:0;95451c90bed2:46367 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:02,277 INFO  [RS:0;95451c90bed2:46367 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:46367.
2024-12-02T18:30:02,278 DEBUG [RS:0;95451c90bed2:46367 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:02,278 DEBUG [RS:0;95451c90bed2:46367 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:02,278 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c466fedace16710fc1f63981c747dae9, disabling compactions & flushes
2024-12-02T18:30:02,278 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:30:02,278 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:30:02,278 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9. after waiting 0 ms
2024-12-02T18:30:02,278 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:30:02,278 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:30:02,278 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:30:02,278 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:30:02,278 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:30:02,278 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c466fedace16710fc1f63981c747dae9 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-02T18:30:02,279 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-02T18:30:02,279 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:30:02,279 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:30:02,279 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:30:02,279 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:30:02,279 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:30:02,279 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-02T18:30:02,279 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c466fedace16710fc1f63981c747dae9=TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.}
2024-12-02T18:30:02,280 DEBUG [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c466fedace16710fc1f63981c747dae9
2024-12-02T18:30:02,284 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/18272759155d48bdaaf84e3127a15c11 is 1080, key is row0029/info:/1733164196167/Put/seqid=0
2024-12-02T18:30:02,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741849_1025 (size=8193)
2024-12-02T18:30:02,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741849_1025 (size=8193)
2024-12-02T18:30:02,292 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/18272759155d48bdaaf84e3127a15c11
2024-12-02T18:30:02,300 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/.tmp/info/18272759155d48bdaaf84e3127a15c11 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/18272759155d48bdaaf84e3127a15c11
2024-12-02T18:30:02,305 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/info/107c1cfb3d784536b27c12ff178548d3 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9./info:regioninfo/1733164115034/Put/seqid=0
2024-12-02T18:30:02,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741850_1026 (size=7016)
2024-12-02T18:30:02,312 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/18272759155d48bdaaf84e3127a15c11, entries=3, sequenceid=48, filesize=8.0 K
2024-12-02T18:30:02,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741850_1026 (size=7016)
2024-12-02T18:30:02,312 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/info/107c1cfb3d784536b27c12ff178548d3
2024-12-02T18:30:02,313 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for c466fedace16710fc1f63981c747dae9 in 35ms, sequenceid=48, compaction requested=true
2024-12-02T18:30:02,314 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad] to archive
2024-12-02T18:30:02,317 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:30:02,320 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70 to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/8c679b6dcf654667ae1ff86965650f70
2024-12-02T18:30:02,322 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18 to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/1a574747bc784a049b127440eb35ef18
2024-12-02T18:30:02,325 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/info/602ba4434bb84627b45d1b191c61cfad
2024-12-02T18:30:02,335 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=95451c90bed2:43055 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-02T18:30:02,339 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8c679b6dcf654667ae1ff86965650f70=12509, 1a574747bc784a049b127440eb35ef18=12509, 602ba4434bb84627b45d1b191c61cfad=12509]
2024-12-02T18:30:02,341 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/ns/13b86b4fa3744c93b8cbd30a1ffe8acd is 43, key is default/ns:d/1733164114300/Put/seqid=0
2024-12-02T18:30:02,349 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/default/TestLogRolling-testSlowSyncLogRolling/c466fedace16710fc1f63981c747dae9/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-02T18:30:02,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741851_1027 (size=5153)
2024-12-02T18:30:02,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741851_1027 (size=5153)
2024-12-02T18:30:02,352 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/ns/13b86b4fa3744c93b8cbd30a1ffe8acd
2024-12-02T18:30:02,352 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:30:02,352 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c466fedace16710fc1f63981c747dae9:
Waiting for close lock at 1733164202277Running coprocessor pre-close hooks at 1733164202278 (+1 ms)Disabling compacts and flushes for region at 1733164202278Disabling writes for close at 1733164202278Obtaining lock to block concurrent updates at 1733164202278Preparing flush snapshotting stores in c466fedace16710fc1f63981c747dae9 at 1733164202278Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733164202278Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9. at 1733164202279 (+1 ms)Flushing c466fedace16710fc1f63981c747dae9/info: creating writer at 1733164202279Flushing c466fedace16710fc1f63981c747dae9/info: appending metadata at 1733164202283 (+4 ms)Flushing c466fedace16710fc1f63981c747dae9/info: closing flushed file at 1733164202284 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1a282c: reopening flushed file at 1733164202299 (+15 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for c466fedace16710fc1f63981c747dae9 in 35ms, sequenceid=48, compaction requested=true at 1733164202313 (+14 ms)Writing region close event to WAL at 1733164202340 (+27 ms)Running coprocessor post-close hooks at 1733164202350 (+10 ms)Closed at 1733164202352 (+2 ms)
2024-12-02T18:30:02,353 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733164114583.c466fedace16710fc1f63981c747dae9.
2024-12-02T18:30:02,376 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/table/e0cf91c654df4cca9bd637ed1394853f is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733164115054/Put/seqid=0
2024-12-02T18:30:02,380 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-02T18:30:02,380 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-02T18:30:02,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741852_1028 (size=5396)
2024-12-02T18:30:02,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741852_1028 (size=5396)
2024-12-02T18:30:02,382 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/table/e0cf91c654df4cca9bd637ed1394853f
2024-12-02T18:30:02,391 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/info/107c1cfb3d784536b27c12ff178548d3 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/info/107c1cfb3d784536b27c12ff178548d3
2024-12-02T18:30:02,400 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/info/107c1cfb3d784536b27c12ff178548d3, entries=10, sequenceid=11, filesize=6.9 K
2024-12-02T18:30:02,402 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/ns/13b86b4fa3744c93b8cbd30a1ffe8acd as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/ns/13b86b4fa3744c93b8cbd30a1ffe8acd
2024-12-02T18:30:02,411 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/ns/13b86b4fa3744c93b8cbd30a1ffe8acd, entries=2, sequenceid=11, filesize=5.0 K
2024-12-02T18:30:02,413 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/.tmp/table/e0cf91c654df4cca9bd637ed1394853f as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/table/e0cf91c654df4cca9bd637ed1394853f
2024-12-02T18:30:02,422 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/table/e0cf91c654df4cca9bd637ed1394853f, entries=2, sequenceid=11, filesize=5.3 K
2024-12-02T18:30:02,423 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false
2024-12-02T18:30:02,429 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-02T18:30:02,430 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:30:02,430 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:02,430 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164202279Running coprocessor pre-close hooks at 1733164202279Disabling compacts and flushes for region at 1733164202279Disabling writes for close at 1733164202279Obtaining lock to block concurrent updates at 1733164202279Preparing flush snapshotting stores in 1588230740 at 1733164202279Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733164202280 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733164202281 (+1 ms)Flushing 1588230740/info: creating writer at 1733164202281Flushing 1588230740/info: appending metadata at 1733164202304 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733164202304Flushing 1588230740/ns: creating writer at 1733164202320 (+16 ms)Flushing 1588230740/ns: appending metadata at 1733164202340 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733164202340Flushing 1588230740/table: creating writer at 1733164202361 (+21 ms)Flushing 1588230740/table: appending metadata at 1733164202375 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733164202375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@236b6134: reopening flushed file at 1733164202390 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e8bb74b: reopening flushed file at 1733164202401 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70bab022: reopening flushed file at 1733164202412 (+11 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false at 1733164202423 (+11 ms)Writing region close event to WAL at 1733164202425 (+2 ms)Running coprocessor post-close hooks at 1733164202430 (+5 ms)Closed at 1733164202430
2024-12-02T18:30:02,431 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:02,480 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,46367,1733164111982; all regions closed.
2024-12-02T18:30:02,482 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,482 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,483 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,483 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,483 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741834_1010 (size=3066)
2024-12-02T18:30:02,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741834_1010 (size=3066)
2024-12-02T18:30:02,489 DEBUG [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs
2024-12-02T18:30:02,489 INFO  [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C46367%2C1733164111982.meta:.meta(num 1733164114098)
2024-12-02T18:30:02,490 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,490 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,490 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,490 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,490 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741846_1022 (size=13040)
2024-12-02T18:30:02,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741846_1022 (size=13040)
2024-12-02T18:30:02,500 DEBUG [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(1256): Moved 5 WAL file(s) to /user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/oldWALs
2024-12-02T18:30:02,500 INFO  [RS:0;95451c90bed2:46367 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C46367%2C1733164111982:(num 1733164182133)
2024-12-02T18:30:02,500 DEBUG [RS:0;95451c90bed2:46367 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:02,501 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:02,501 INFO  [RS:0;95451c90bed2:46367 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:02,501 INFO  [RS:0;95451c90bed2:46367 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:02,501 INFO  [RS:0;95451c90bed2:46367 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:02,501 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:02,502 INFO  [RS:0;95451c90bed2:46367 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46367
2024-12-02T18:30:02,517 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,46367,1733164111982
2024-12-02T18:30:02,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:02,517 INFO  [RS:0;95451c90bed2:46367 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:02,527 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,46367,1733164111982]
2024-12-02T18:30:02,538 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,46367,1733164111982 already deleted, retry=false
2024-12-02T18:30:02,538 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,46367,1733164111982 expired; onlineServers=0
2024-12-02T18:30:02,538 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,43055,1733164111212' *****
2024-12-02T18:30:02,538 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:30:02,538 INFO  [M:0;95451c90bed2:43055 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:02,538 INFO  [M:0;95451c90bed2:43055 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:02,538 DEBUG [M:0;95451c90bed2:43055 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:30:02,539 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:30:02,539 DEBUG [M:0;95451c90bed2:43055 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:30:02,539 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164113145 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164113145,5,FailOnTimeoutGroup]
2024-12-02T18:30:02,539 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164113142 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164113142,5,FailOnTimeoutGroup]
2024-12-02T18:30:02,539 INFO  [M:0;95451c90bed2:43055 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:02,539 INFO  [M:0;95451c90bed2:43055 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:02,539 DEBUG [M:0;95451c90bed2:43055 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:30:02,539 INFO  [M:0;95451c90bed2:43055 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:30:02,539 INFO  [M:0;95451c90bed2:43055 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:02,540 INFO  [M:0;95451c90bed2:43055 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:30:02,540 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:30:02,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:02,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:02,548 DEBUG [M:0;95451c90bed2:43055 {}] zookeeper.ZKUtil(347): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:30:02,548 WARN  [M:0;95451c90bed2:43055 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:30:02,549 INFO  [M:0;95451c90bed2:43055 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/.lastflushedseqids
2024-12-02T18:30:02,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741853_1029 (size=130)
2024-12-02T18:30:02,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741853_1029 (size=130)
2024-12-02T18:30:02,563 INFO  [M:0;95451c90bed2:43055 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:30:02,563 INFO  [M:0;95451c90bed2:43055 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:30:02,563 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:02,563 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:02,563 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:02,563 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:02,563 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:02,564 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.01 KB heapSize=29.18 KB
2024-12-02T18:30:02,585 DEBUG [M:0;95451c90bed2:43055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bfdf03fcac249f0949a43137a708697 is 82, key is hbase:meta,,1/info:regioninfo/1733164114181/Put/seqid=0
2024-12-02T18:30:02,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741854_1030 (size=5672)
2024-12-02T18:30:02,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741854_1030 (size=5672)
2024-12-02T18:30:02,591 INFO  [M:0;95451c90bed2:43055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bfdf03fcac249f0949a43137a708697
2024-12-02T18:30:02,620 DEBUG [M:0;95451c90bed2:43055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63747a4cd5c94118b218d7a4b1227385 is 765, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733164115062/Put/seqid=0
2024-12-02T18:30:02,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741855_1031 (size=6246)
2024-12-02T18:30:02,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741855_1031 (size=6246)
2024-12-02T18:30:02,626 INFO  [M:0;95451c90bed2:43055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.41 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63747a4cd5c94118b218d7a4b1227385
2024-12-02T18:30:02,628 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:02,628 DEBUG [pool-51-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46367-0x101974d8dfd0001, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:02,628 INFO  [RS:0;95451c90bed2:46367 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:02,628 INFO  [RS:0;95451c90bed2:46367 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,46367,1733164111982; zookeeper connection closed.
2024-12-02T18:30:02,629 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40dc7e36 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40dc7e36
2024-12-02T18:30:02,629 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:30:02,634 INFO  [M:0;95451c90bed2:43055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63747a4cd5c94118b218d7a4b1227385
2024-12-02T18:30:02,652 DEBUG [M:0;95451c90bed2:43055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9541e9ddd6d14a88bf3e70d87d9cfbf0 is 69, key is 95451c90bed2,46367,1733164111982/rs:state/1733164113216/Put/seqid=0
2024-12-02T18:30:02,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741856_1032 (size=5156)
2024-12-02T18:30:02,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741856_1032 (size=5156)
2024-12-02T18:30:02,659 INFO  [M:0;95451c90bed2:43055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9541e9ddd6d14a88bf3e70d87d9cfbf0
2024-12-02T18:30:02,687 DEBUG [M:0;95451c90bed2:43055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2ab8daa188db42be90d82c6ffd9b0e65 is 52, key is load_balancer_on/state:d/1733164114560/Put/seqid=0
2024-12-02T18:30:02,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741857_1033 (size=5056)
2024-12-02T18:30:02,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741857_1033 (size=5056)
2024-12-02T18:30:02,694 INFO  [M:0;95451c90bed2:43055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2ab8daa188db42be90d82c6ffd9b0e65
2024-12-02T18:30:02,702 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bfdf03fcac249f0949a43137a708697 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5bfdf03fcac249f0949a43137a708697
2024-12-02T18:30:02,709 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5bfdf03fcac249f0949a43137a708697, entries=8, sequenceid=59, filesize=5.5 K
2024-12-02T18:30:02,710 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63747a4cd5c94118b218d7a4b1227385 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/63747a4cd5c94118b218d7a4b1227385
2024-12-02T18:30:02,717 INFO  [M:0;95451c90bed2:43055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 63747a4cd5c94118b218d7a4b1227385
2024-12-02T18:30:02,717 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/63747a4cd5c94118b218d7a4b1227385, entries=6, sequenceid=59, filesize=6.1 K
2024-12-02T18:30:02,719 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9541e9ddd6d14a88bf3e70d87d9cfbf0 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9541e9ddd6d14a88bf3e70d87d9cfbf0
2024-12-02T18:30:02,725 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9541e9ddd6d14a88bf3e70d87d9cfbf0, entries=1, sequenceid=59, filesize=5.0 K
2024-12-02T18:30:02,726 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2ab8daa188db42be90d82c6ffd9b0e65 as hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2ab8daa188db42be90d82c6ffd9b0e65
2024-12-02T18:30:02,733 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2ab8daa188db42be90d82c6ffd9b0e65, entries=1, sequenceid=59, filesize=4.9 K
2024-12-02T18:30:02,734 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=59, compaction requested=false
2024-12-02T18:30:02,736 INFO  [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:02,736 DEBUG [M:0;95451c90bed2:43055 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164202563Disabling compacts and flushes for region at 1733164202563Disabling writes for close at 1733164202563Obtaining lock to block concurrent updates at 1733164202564 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164202564Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23564, getHeapSize=29816, getOffHeapSize=0, getCellsCount=70 at 1733164202564Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164202565 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164202565Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164202584 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164202584Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164202598 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164202619 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164202619Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164202634 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164202652 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164202652Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164202667 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164202686 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164202686Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5942b215: reopening flushed file at 1733164202701 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@99038bb: reopening flushed file at 1733164202709 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11e9198a: reopening flushed file at 1733164202717 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb1b1b3: reopening flushed file at 1733164202725 (+8 ms)Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=59, compaction requested=false at 1733164202734 (+9 ms)Writing region close event to WAL at 1733164202736 (+2 ms)Closed at 1733164202736
2024-12-02T18:30:02,737 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,737 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,737 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,737 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,737 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:02,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741830_1006 (size=27961)
2024-12-02T18:30:02,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36031 is added to blk_1073741830_1006 (size=27961)
2024-12-02T18:30:02,740 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:02,740 INFO  [M:0;95451c90bed2:43055 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:30:02,741 INFO  [M:0;95451c90bed2:43055 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43055
2024-12-02T18:30:02,741 INFO  [M:0;95451c90bed2:43055 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:02,853 INFO  [M:0;95451c90bed2:43055 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43055-0x101974d8dfd0000, quorum=127.0.0.1:50294, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:02,857 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:02,859 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:02,860 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:02,860 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:02,860 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:02,863 WARN  [BP-1810398651-172.17.0.3-1733164106500 heartbeating to localhost/127.0.0.1:36351 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:02,863 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:02,863 WARN  [BP-1810398651-172.17.0.3-1733164106500 heartbeating to localhost/127.0.0.1:36351 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1810398651-172.17.0.3-1733164106500 (Datanode Uuid e4c71843-62f8-47ee-8357-96edf56da42e) service to localhost/127.0.0.1:36351
2024-12-02T18:30:02,863 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:02,864 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data3/current/BP-1810398651-172.17.0.3-1733164106500 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:02,864 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data4/current/BP-1810398651-172.17.0.3-1733164106500 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:02,865 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:02,866 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:02,867 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:02,867 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:02,867 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:02,867 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:02,868 WARN  [BP-1810398651-172.17.0.3-1733164106500 heartbeating to localhost/127.0.0.1:36351 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:02,868 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:02,868 WARN  [BP-1810398651-172.17.0.3-1733164106500 heartbeating to localhost/127.0.0.1:36351 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1810398651-172.17.0.3-1733164106500 (Datanode Uuid f4926761-ad1f-4d40-b334-fc1ca3f97dc8) service to localhost/127.0.0.1:36351
2024-12-02T18:30:02,868 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:02,869 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data1/current/BP-1810398651-172.17.0.3-1733164106500 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:02,869 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/cluster_261d82cc-474e-c21c-512c-4c889b040309/data/data2/current/BP-1810398651-172.17.0.3-1733164106500 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:02,869 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:02,876 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:02,877 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:02,877 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:02,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:02,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:02,886 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:30:02,918 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:30:02,926 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/95451c90bed2:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36351
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36351
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/95451c90bed2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36351 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36351
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36351
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36351 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36351
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/95451c90bed2:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36351 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/95451c90bed2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@5829c7be
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=179 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=5196 (was 4875) - AvailableMemoryMB LEAK? -
2024-12-02T18:30:02,932 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=179, ProcessCount=11, AvailableMemoryMB=5196
2024-12-02T18:30:02,932 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:30:02,932 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.log.dir so I do NOT create it in target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974
2024-12-02T18:30:02,932 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bbc4404a-9fc6-c458-3097-7d10ee03662a/hadoop.tmp.dir so I do NOT create it in target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04, deleteOnExit=true
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/test.cache.data in system properties and HBase conf
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:30:02,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:30:02,933 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:30:02,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:30:02,947 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:03,269 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:03,274 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:03,276 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:03,276 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:03,276 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:03,278 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:03,278 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:03,279 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:03,298 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:03,371 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51b7943b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/java.io.tmpdir/jetty-localhost-40201-hadoop-hdfs-3_4_1-tests_jar-_-any-12476005107616902525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:03,372 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:40201}
2024-12-02T18:30:03,372 INFO  [Time-limited test {}] server.Server(415): Started @98758ms
2024-12-02T18:30:03,383 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:03,643 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:03,648 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:03,649 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:03,649 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:03,649 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:30:03,650 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:03,650 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:03,746 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b9e849{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/java.io.tmpdir/jetty-localhost-44003-hadoop-hdfs-3_4_1-tests_jar-_-any-1177303007985302265/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:03,747 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:44003}
2024-12-02T18:30:03,747 INFO  [Time-limited test {}] server.Server(415): Started @99133ms
2024-12-02T18:30:03,748 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:03,781 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:03,785 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:03,786 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:03,786 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:03,786 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:03,787 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:03,787 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:03,879 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e721f3d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/java.io.tmpdir/jetty-localhost-33771-hadoop-hdfs-3_4_1-tests_jar-_-any-5274592042062249160/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:03,880 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:33771}
2024-12-02T18:30:03,880 INFO  [Time-limited test {}] server.Server(415): Started @99266ms
2024-12-02T18:30:03,882 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:04,934 WARN  [Thread-445 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data1/current/BP-1991308746-172.17.0.3-1733164202958/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:04,936 WARN  [Thread-446 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data2/current/BP-1991308746-172.17.0.3-1733164202958/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:04,970 WARN  [Thread-409 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:04,974 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1503ca3982ee70a9 with lease ID 0x8f3ff36390fbf727: Processing first storage report for DS-fd026f35-a9a9-4903-9018-b12f74af5a51 from datanode DatanodeRegistration(127.0.0.1:37793, datanodeUuid=451b7e0f-858f-45e1-b320-7850d33e9cb8, infoPort=36287, infoSecurePort=0, ipcPort=38381, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958)
2024-12-02T18:30:04,974 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1503ca3982ee70a9 with lease ID 0x8f3ff36390fbf727: from storage DS-fd026f35-a9a9-4903-9018-b12f74af5a51 node DatanodeRegistration(127.0.0.1:37793, datanodeUuid=451b7e0f-858f-45e1-b320-7850d33e9cb8, infoPort=36287, infoSecurePort=0, ipcPort=38381, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:30:04,974 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1503ca3982ee70a9 with lease ID 0x8f3ff36390fbf727: Processing first storage report for DS-eec9716f-8148-420c-aabd-053486e3ab45 from datanode DatanodeRegistration(127.0.0.1:37793, datanodeUuid=451b7e0f-858f-45e1-b320-7850d33e9cb8, infoPort=36287, infoSecurePort=0, ipcPort=38381, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958)
2024-12-02T18:30:04,974 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1503ca3982ee70a9 with lease ID 0x8f3ff36390fbf727: from storage DS-eec9716f-8148-420c-aabd-053486e3ab45 node DatanodeRegistration(127.0.0.1:37793, datanodeUuid=451b7e0f-858f-45e1-b320-7850d33e9cb8, infoPort=36287, infoSecurePort=0, ipcPort=38381, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:05,060 WARN  [Thread-456 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data3/current/BP-1991308746-172.17.0.3-1733164202958/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:05,060 WARN  [Thread-457 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data4/current/BP-1991308746-172.17.0.3-1733164202958/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:05,077 WARN  [Thread-432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:05,080 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x286d168d69f52f9d with lease ID 0x8f3ff36390fbf728: Processing first storage report for DS-dd88540e-f8c6-4054-b4bd-3c48a508273f from datanode DatanodeRegistration(127.0.0.1:39673, datanodeUuid=494a9935-39ed-4fb4-a254-0e35f408daeb, infoPort=46517, infoSecurePort=0, ipcPort=37679, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958)
2024-12-02T18:30:05,080 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x286d168d69f52f9d with lease ID 0x8f3ff36390fbf728: from storage DS-dd88540e-f8c6-4054-b4bd-3c48a508273f node DatanodeRegistration(127.0.0.1:39673, datanodeUuid=494a9935-39ed-4fb4-a254-0e35f408daeb, infoPort=46517, infoSecurePort=0, ipcPort=37679, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:05,080 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x286d168d69f52f9d with lease ID 0x8f3ff36390fbf728: Processing first storage report for DS-f0e75df7-d28d-4211-b11d-940678ef5655 from datanode DatanodeRegistration(127.0.0.1:39673, datanodeUuid=494a9935-39ed-4fb4-a254-0e35f408daeb, infoPort=46517, infoSecurePort=0, ipcPort=37679, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958)
2024-12-02T18:30:05,080 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x286d168d69f52f9d with lease ID 0x8f3ff36390fbf728: from storage DS-f0e75df7-d28d-4211-b11d-940678ef5655 node DatanodeRegistration(127.0.0.1:39673, datanodeUuid=494a9935-39ed-4fb4-a254-0e35f408daeb, infoPort=46517, infoSecurePort=0, ipcPort=37679, storageInfo=lv=-57;cid=testClusterID;nsid=126719086;c=1733164202958), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:05,117 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974
2024-12-02T18:30:05,120 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/zookeeper_0, clientPort=54967, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:30:05,121 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54967
2024-12-02T18:30:05,122 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,124 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:05,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:05,135 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c with version=8
2024-12-02T18:30:05,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:30:05,137 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:30:05,138 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:05,139 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46319
2024-12-02T18:30:05,140 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46319 connecting to ZooKeeper ensemble=127.0.0.1:54967
2024-12-02T18:30:05,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463190x0, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:05,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46319-0x101974effeb0000 connected
2024-12-02T18:30:05,269 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,270 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:05,273 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c, hbase.cluster.distributed=false
2024-12-02T18:30:05,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:05,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46319
2024-12-02T18:30:05,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46319
2024-12-02T18:30:05,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46319
2024-12-02T18:30:05,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46319
2024-12-02T18:30:05,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46319
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:05,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:30:05,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:05,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32871
2024-12-02T18:30:05,294 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32871 connecting to ZooKeeper ensemble=127.0.0.1:54967
2024-12-02T18:30:05,295 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,297 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328710x0, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:05,312 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328710x0, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:05,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32871-0x101974effeb0001 connected
2024-12-02T18:30:05,312 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:30:05,313 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:30:05,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:30:05,315 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:05,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32871
2024-12-02T18:30:05,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32871
2024-12-02T18:30:05,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32871
2024-12-02T18:30:05,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32871
2024-12-02T18:30:05,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32871
2024-12-02T18:30:05,333 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:46319
2024-12-02T18:30:05,336 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:05,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:05,349 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:05,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,359 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:30:05,360 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,46319,1733164205137 from backup master directory
2024-12-02T18:30:05,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:05,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:05,369 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:05,369 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,379 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/hbase.id] with ID: 3912c4fc-88f9-4a66-be1b-882831250c80
2024-12-02T18:30:05,379 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/.tmp/hbase.id
2024-12-02T18:30:05,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:05,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:05,389 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/.tmp/hbase.id]:[hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/hbase.id]
2024-12-02T18:30:05,409 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:05,409 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:30:05,412 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-12-02T18:30:05,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:05,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:05,435 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:30:05,436 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:30:05,437 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:05,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:05,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:05,447 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store
2024-12-02T18:30:05,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:05,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:05,455 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:05,456 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:05,456 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:05,456 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:05,456 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:05,456 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:05,456 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:05,456 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164205456Disabling compacts and flushes for region at 1733164205456Disabling writes for close at 1733164205456Writing region close event to WAL at 1733164205456Closed at 1733164205456
2024-12-02T18:30:05,457 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/.initializing
2024-12-02T18:30:05,457 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/WALs/95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,461 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C46319%2C1733164205137, suffix=, logDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/WALs/95451c90bed2,46319,1733164205137, archiveDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/oldWALs, maxLogs=10
2024-12-02T18:30:05,461 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46319%2C1733164205137.1733164205461
2024-12-02T18:30:05,467 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/WALs/95451c90bed2,46319,1733164205137/95451c90bed2%2C46319%2C1733164205137.1733164205461
2024-12-02T18:30:05,468 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46517:46517),(127.0.0.1/127.0.0.1:36287:36287)]
2024-12-02T18:30:05,472 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:05,472 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:05,472 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,472 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,474 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:05,476 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:30:05,476 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,477 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:05,477 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:05,478 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:30:05,478 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,479 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:05,479 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:05,481 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:30:05,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,482 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:05,482 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:05,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:30:05,484 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,485 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:05,485 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,486 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,486 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,487 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,487 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,488 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:30:05,489 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:05,492 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:05,493 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=730188, jitterRate=-0.07151907682418823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:30:05,495 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164205473Initializing all the Stores at 1733164205474 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164205474Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164205474Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164205474Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164205474Cleaning up temporary data from old regions at 1733164205487 (+13 ms)Region opened successfully at 1733164205494 (+7 ms)
2024-12-02T18:30:05,495 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:30:05,500 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461fa369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:05,502 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:30:05,502 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:30:05,502 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:30:05,502 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:30:05,503 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:30:05,504 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:30:05,504 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:30:05,510 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:30:05,512 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:30:05,521 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:30:05,522 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:30:05,523 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:30:05,532 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:30:05,532 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:30:05,534 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:30:05,542 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:30:05,543 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:30:05,553 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:30:05,555 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:30:05,563 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:30:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,575 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,46319,1733164205137, sessionid=0x101974effeb0000, setting cluster-up flag (Was=false)
2024-12-02T18:30:05,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,626 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:30:05,628 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:05,679 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:30:05,681 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,46319,1733164205137
2024-12-02T18:30:05,682 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:30:05,684 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:05,684 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:30:05,684 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:30:05,684 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,46319,1733164205137
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:05,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,687 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164235687
2024-12-02T18:30:05,687 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:30:05,688 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:05,688 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:30:05,689 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:30:05,689 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:30:05,689 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:30:05,689 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164205689,5,FailOnTimeoutGroup]
2024-12-02T18:30:05,689 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164205689,5,FailOnTimeoutGroup]
2024-12-02T18:30:05,689 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,689 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:30:05,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,690 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,690 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:30:05,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:05,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:05,698 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:30:05,698 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c
2024-12-02T18:30:05,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:05,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:05,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:05,707 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:30:05,708 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:30:05,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,709 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:05,709 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:30:05,711 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:30:05,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,712 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:05,712 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:30:05,714 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:30:05,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,715 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:05,715 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:30:05,717 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:30:05,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:05,717 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:05,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:30:05,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740
2024-12-02T18:30:05,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740
2024-12-02T18:30:05,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:30:05,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:30:05,722 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:30:05,723 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(746): ClusterId : 3912c4fc-88f9-4a66-be1b-882831250c80
2024-12-02T18:30:05,723 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:30:05,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:30:05,727 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:05,727 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752516, jitterRate=-0.0431269109249115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:30:05,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164205705Initializing all the Stores at 1733164205706 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164205706Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164205706Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164205706Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164205706Cleaning up temporary data from old regions at 1733164205721 (+15 ms)Region opened successfully at 1733164205728 (+7 ms)
2024-12-02T18:30:05,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:30:05,729 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:30:05,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:30:05,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:30:05,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:30:05,731 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:05,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164205729Disabling compacts and flushes for region at 1733164205729Disabling writes for close at 1733164205729Writing region close event to WAL at 1733164205731 (+2 ms)Closed at 1733164205731
2024-12-02T18:30:05,733 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:30:05,733 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:30:05,733 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:05,733 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:30:05,734 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:30:05,736 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:30:05,737 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:30:05,744 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:30:05,744 DEBUG [RS:0;95451c90bed2:32871 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5b57cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:05,761 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:32871
2024-12-02T18:30:05,761 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:30:05,761 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:30:05,761 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:30:05,762 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,46319,1733164205137 with port=32871, startcode=1733164205292
2024-12-02T18:30:05,762 DEBUG [RS:0;95451c90bed2:32871 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:30:05,765 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45049, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:30:05,766 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,766 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46319 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,768 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c
2024-12-02T18:30:05,768 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45303
2024-12-02T18:30:05,768 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:30:05,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:05,780 DEBUG [RS:0;95451c90bed2:32871 {}] zookeeper.ZKUtil(111): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,780 WARN  [RS:0;95451c90bed2:32871 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:05,780 INFO  [RS:0;95451c90bed2:32871 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:05,780 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/WALs/95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,781 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,32871,1733164205292]
2024-12-02T18:30:05,785 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:30:05,787 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:30:05,787 INFO  [RS:0;95451c90bed2:32871 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:30:05,787 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,787 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:30:05,789 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:30:05,789 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,789 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:05,790 DEBUG [RS:0;95451c90bed2:32871 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,791 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,32871,1733164205292-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:30:05,806 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:30:05,806 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,32871,1733164205292-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,806 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,806 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.Replication(171): 95451c90bed2,32871,1733164205292 started
2024-12-02T18:30:05,819 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:05,819 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,32871,1733164205292, RpcServer on 95451c90bed2/172.17.0.3:32871, sessionid=0x101974effeb0001
2024-12-02T18:30:05,819 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:30:05,819 DEBUG [RS:0;95451c90bed2:32871 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,819 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,32871,1733164205292'
2024-12-02T18:30:05,819 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:30:05,820 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:30:05,820 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:30:05,820 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:30:05,821 DEBUG [RS:0;95451c90bed2:32871 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,32871,1733164205292
2024-12-02T18:30:05,821 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,32871,1733164205292'
2024-12-02T18:30:05,821 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:30:05,821 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:30:05,822 DEBUG [RS:0;95451c90bed2:32871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:30:05,822 INFO  [RS:0;95451c90bed2:32871 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:30:05,822 INFO  [RS:0;95451c90bed2:32871 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:30:05,888 WARN  [95451c90bed2:46319 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:30:05,924 INFO  [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C32871%2C1733164205292, suffix=, logDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/WALs/95451c90bed2,32871,1733164205292, archiveDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/oldWALs, maxLogs=32
2024-12-02T18:30:05,926 INFO  [RS:0;95451c90bed2:32871 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C32871%2C1733164205292.1733164205926
2024-12-02T18:30:05,932 INFO  [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/WALs/95451c90bed2,32871,1733164205292/95451c90bed2%2C32871%2C1733164205292.1733164205926
2024-12-02T18:30:05,932 DEBUG [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36287:36287),(127.0.0.1/127.0.0.1:46517:46517)]
2024-12-02T18:30:06,138 DEBUG [95451c90bed2:46319 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:30:06,139 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,32871,1733164205292
2024-12-02T18:30:06,140 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,32871,1733164205292, state=OPENING
2024-12-02T18:30:06,200 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:30:06,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:06,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:06,212 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:30:06,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:06,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:06,212 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,32871,1733164205292}]
2024-12-02T18:30:06,365 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:30:06,367 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41231, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:30:06,371 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:30:06,372 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:06,374 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C32871%2C1733164205292.meta, suffix=.meta, logDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/WALs/95451c90bed2,32871,1733164205292, archiveDir=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/oldWALs, maxLogs=32
2024-12-02T18:30:06,376 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C32871%2C1733164205292.meta.1733164206375.meta
2024-12-02T18:30:06,381 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/WALs/95451c90bed2,32871,1733164205292/95451c90bed2%2C32871%2C1733164205292.meta.1733164206375.meta
2024-12-02T18:30:06,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36287:36287),(127.0.0.1/127.0.0.1:46517:46517)]
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:30:06,383 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:30:06,383 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:30:06,385 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:30:06,386 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:30:06,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:06,386 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:06,387 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:30:06,387 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:30:06,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:06,388 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:06,388 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:30:06,389 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:30:06,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:06,390 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:06,390 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:30:06,391 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:30:06,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:06,391 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:06,391 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:30:06,392 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740
2024-12-02T18:30:06,393 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740
2024-12-02T18:30:06,395 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:30:06,395 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:30:06,395 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:30:06,397 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:30:06,398 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825676, jitterRate=0.04990231990814209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:30:06,398 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:30:06,399 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164206383Writing region info on filesystem at 1733164206383Initializing all the Stores at 1733164206384 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164206384Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164206385 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164206385Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164206385Cleaning up temporary data from old regions at 1733164206395 (+10 ms)Running coprocessor post-open hooks at 1733164206398 (+3 ms)Region opened successfully at 1733164206399 (+1 ms)
2024-12-02T18:30:06,401 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164206365
2024-12-02T18:30:06,404 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:30:06,404 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:30:06,405 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,32871,1733164205292
2024-12-02T18:30:06,406 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,32871,1733164205292, state=OPEN
2024-12-02T18:30:06,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:30:06,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:30:06,448 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,32871,1733164205292
2024-12-02T18:30:06,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:06,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:06,451 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:30:06,451 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,32871,1733164205292 in 236 msec
2024-12-02T18:30:06,454 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:30:06,455 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 718 msec
2024-12-02T18:30:06,455 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:06,456 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:30:06,457 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:30:06,457 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,32871,1733164205292, seqNum=-1]
2024-12-02T18:30:06,458 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:30:06,459 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56237, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:30:06,467 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 782 msec
2024-12-02T18:30:06,468 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164206468, completionTime=-1
2024-12-02T18:30:06,468 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:30:06,468 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:30:06,470 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:30:06,470 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164266470
2024-12-02T18:30:06,470 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164326470
2024-12-02T18:30:06,470 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-02T18:30:06,470 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,471 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,471 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,471 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:46319, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,471 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,471 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,473 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.108sec
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:30:06,477 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:30:06,478 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:30:06,478 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:30:06,481 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:30:06,481 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:30:06,481 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46319,1733164205137-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:06,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9379ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:06,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,46319,-1 for getting cluster id
2024-12-02T18:30:06,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:30:06,525 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3912c4fc-88f9-4a66-be1b-882831250c80'
2024-12-02T18:30:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:30:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3912c4fc-88f9-4a66-be1b-882831250c80"

2024-12-02T18:30:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111b08da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,46319,-1]
2024-12-02T18:30:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:30:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,529 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:30:06,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aadf114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:06,531 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:30:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,32871,1733164205292, seqNum=-1]
2024-12-02T18:30:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:30:06,536 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:30:06,539 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,46319,1733164205137
2024-12-02T18:30:06,540 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:06,543 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:30:06,543 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:30:06,544 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:06,544 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:06,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,544 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:30:06,544 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2044181255, stopped=false
2024-12-02T18:30:06,545 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,46319,1733164205137
2024-12-02T18:30:06,545 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:30:06,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:06,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:06,564 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:06,564 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:06,564 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:06,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,564 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,32871,1733164205292' *****
2024-12-02T18:30:06,564 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:30:06,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:06,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:30:06,565 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,32871,1733164205292
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:32871.
2024-12-02T18:30:06,565 DEBUG [RS:0;95451c90bed2:32871 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:06,565 DEBUG [RS:0;95451c90bed2:32871 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:30:06,565 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:30:06,566 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:30:06,566 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:30:06,566 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-02T18:30:06,566 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-02T18:30:06,566 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:30:06,566 DEBUG [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-02T18:30:06,566 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:30:06,566 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:30:06,566 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:30:06,566 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:30:06,566 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-02T18:30:06,583 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/.tmp/ns/5d4da347cc264f5b817805416fe1d708 is 43, key is default/ns:d/1733164206460/Put/seqid=0
2024-12-02T18:30:06,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741835_1011 (size=5153)
2024-12-02T18:30:06,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741835_1011 (size=5153)
2024-12-02T18:30:06,592 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/.tmp/ns/5d4da347cc264f5b817805416fe1d708
2024-12-02T18:30:06,603 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/.tmp/ns/5d4da347cc264f5b817805416fe1d708 as hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/ns/5d4da347cc264f5b817805416fe1d708
2024-12-02T18:30:06,612 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/ns/5d4da347cc264f5b817805416fe1d708, entries=2, sequenceid=6, filesize=5.0 K
2024-12-02T18:30:06,616 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false
2024-12-02T18:30:06,616 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:30:06,623 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-02T18:30:06,624 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:30:06,624 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:06,624 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164206566Running coprocessor pre-close hooks at 1733164206566Disabling compacts and flushes for region at 1733164206566Disabling writes for close at 1733164206566Obtaining lock to block concurrent updates at 1733164206566Preparing flush snapshotting stores in 1588230740 at 1733164206566Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733164206567 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733164206568 (+1 ms)Flushing 1588230740/ns: creating writer at 1733164206568Flushing 1588230740/ns: appending metadata at 1733164206583 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733164206583Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@514357dc: reopening flushed file at 1733164206602 (+19 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false at 1733164206616 (+14 ms)Writing region close event to WAL at 1733164206618 (+2 ms)Running coprocessor post-close hooks at 1733164206624 (+6 ms)Closed at 1733164206624
2024-12-02T18:30:06,624 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:06,766 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,32871,1733164205292; all regions closed.
2024-12-02T18:30:06,767 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,767 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,767 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,767 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,768 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741834_1010 (size=1152)
2024-12-02T18:30:06,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741834_1010 (size=1152)
2024-12-02T18:30:06,773 DEBUG [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/oldWALs
2024-12-02T18:30:06,773 INFO  [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C32871%2C1733164205292.meta:.meta(num 1733164206375)
2024-12-02T18:30:06,774 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,774 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,774 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,774 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,774 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:06,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741833_1009 (size=93)
2024-12-02T18:30:06,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741833_1009 (size=93)
2024-12-02T18:30:06,779 DEBUG [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/oldWALs
2024-12-02T18:30:06,779 INFO  [RS:0;95451c90bed2:32871 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C32871%2C1733164205292:(num 1733164205926)
2024-12-02T18:30:06,779 DEBUG [RS:0;95451c90bed2:32871 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:06,779 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:06,780 INFO  [RS:0;95451c90bed2:32871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:06,780 INFO  [RS:0;95451c90bed2:32871 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:06,780 INFO  [RS:0;95451c90bed2:32871 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:06,780 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:06,780 INFO  [RS:0;95451c90bed2:32871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32871
2024-12-02T18:30:06,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,32871,1733164205292
2024-12-02T18:30:06,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:06,790 INFO  [RS:0;95451c90bed2:32871 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:06,800 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,32871,1733164205292]
2024-12-02T18:30:06,811 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,32871,1733164205292 already deleted, retry=false
2024-12-02T18:30:06,811 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,32871,1733164205292 expired; onlineServers=0
2024-12-02T18:30:06,811 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,46319,1733164205137' *****
2024-12-02T18:30:06,811 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:30:06,811 INFO  [M:0;95451c90bed2:46319 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:06,811 INFO  [M:0;95451c90bed2:46319 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:06,811 DEBUG [M:0;95451c90bed2:46319 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:30:06,812 DEBUG [M:0;95451c90bed2:46319 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:30:06,812 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:30:06,812 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164205689 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164205689,5,FailOnTimeoutGroup]
2024-12-02T18:30:06,812 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164205689 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164205689,5,FailOnTimeoutGroup]
2024-12-02T18:30:06,812 INFO  [M:0;95451c90bed2:46319 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:06,812 INFO  [M:0;95451c90bed2:46319 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:06,812 DEBUG [M:0;95451c90bed2:46319 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:30:06,812 INFO  [M:0;95451c90bed2:46319 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:30:06,812 INFO  [M:0;95451c90bed2:46319 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:06,812 INFO  [M:0;95451c90bed2:46319 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:30:06,812 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:30:06,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:06,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:06,822 DEBUG [M:0;95451c90bed2:46319 {}] zookeeper.ZKUtil(347): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:30:06,823 WARN  [M:0;95451c90bed2:46319 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:30:06,823 INFO  [M:0;95451c90bed2:46319 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/.lastflushedseqids
2024-12-02T18:30:06,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741836_1012 (size=99)
2024-12-02T18:30:06,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741836_1012 (size=99)
2024-12-02T18:30:06,835 INFO  [M:0;95451c90bed2:46319 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:30:06,835 INFO  [M:0;95451c90bed2:46319 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:30:06,835 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:06,835 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:06,835 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:06,835 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:06,835 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:06,836 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-02T18:30:06,856 DEBUG [M:0;95451c90bed2:46319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50aeaaf7a89c438aadf354fca204bd3f is 82, key is hbase:meta,,1/info:regioninfo/1733164206405/Put/seqid=0
2024-12-02T18:30:06,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741837_1013 (size=5672)
2024-12-02T18:30:06,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741837_1013 (size=5672)
2024-12-02T18:30:06,871 INFO  [M:0;95451c90bed2:46319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50aeaaf7a89c438aadf354fca204bd3f
2024-12-02T18:30:06,894 DEBUG [M:0;95451c90bed2:46319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522cdc26c6604de1b2926aa5c1863b53 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733164206466/Put/seqid=0
2024-12-02T18:30:06,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741838_1014 (size=5275)
2024-12-02T18:30:06,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741838_1014 (size=5275)
2024-12-02T18:30:06,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:06,901 INFO  [RS:0;95451c90bed2:32871 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:06,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32871-0x101974effeb0001, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:06,901 INFO  [RS:0;95451c90bed2:32871 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,32871,1733164205292; zookeeper connection closed.
2024-12-02T18:30:06,901 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2bc9b7c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2bc9b7c9
2024-12-02T18:30:06,901 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:30:07,301 INFO  [M:0;95451c90bed2:46319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522cdc26c6604de1b2926aa5c1863b53
2024-12-02T18:30:07,323 DEBUG [M:0;95451c90bed2:46319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea9ddb207ab43d38a4416a8f1db4c78 is 69, key is 95451c90bed2,32871,1733164205292/rs:state/1733164205766/Put/seqid=0
2024-12-02T18:30:07,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741839_1015 (size=5156)
2024-12-02T18:30:07,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741839_1015 (size=5156)
2024-12-02T18:30:07,330 INFO  [M:0;95451c90bed2:46319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea9ddb207ab43d38a4416a8f1db4c78
2024-12-02T18:30:07,352 DEBUG [M:0;95451c90bed2:46319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fa964cc44f13465aa28e55c5ea987425 is 52, key is load_balancer_on/state:d/1733164206542/Put/seqid=0
2024-12-02T18:30:07,354 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:07,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:07,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741840_1016 (size=5056)
2024-12-02T18:30:07,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741840_1016 (size=5056)
2024-12-02T18:30:07,363 INFO  [M:0;95451c90bed2:46319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fa964cc44f13465aa28e55c5ea987425
2024-12-02T18:30:07,372 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/50aeaaf7a89c438aadf354fca204bd3f as hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/50aeaaf7a89c438aadf354fca204bd3f
2024-12-02T18:30:07,379 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/50aeaaf7a89c438aadf354fca204bd3f, entries=8, sequenceid=29, filesize=5.5 K
2024-12-02T18:30:07,381 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/522cdc26c6604de1b2926aa5c1863b53 as hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/522cdc26c6604de1b2926aa5c1863b53
2024-12-02T18:30:07,391 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/522cdc26c6604de1b2926aa5c1863b53, entries=3, sequenceid=29, filesize=5.2 K
2024-12-02T18:30:07,393 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea9ddb207ab43d38a4416a8f1db4c78 as hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ea9ddb207ab43d38a4416a8f1db4c78
2024-12-02T18:30:07,402 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ea9ddb207ab43d38a4416a8f1db4c78, entries=1, sequenceid=29, filesize=5.0 K
2024-12-02T18:30:07,403 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fa964cc44f13465aa28e55c5ea987425 as hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fa964cc44f13465aa28e55c5ea987425
2024-12-02T18:30:07,409 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45303/user/jenkins/test-data/285f16da-9780-f981-8702-ea04d83e520c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fa964cc44f13465aa28e55c5ea987425, entries=1, sequenceid=29, filesize=4.9 K
2024-12-02T18:30:07,411 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 576ms, sequenceid=29, compaction requested=false
2024-12-02T18:30:07,412 INFO  [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:07,412 DEBUG [M:0;95451c90bed2:46319 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164206835Disabling compacts and flushes for region at 1733164206835Disabling writes for close at 1733164206835Obtaining lock to block concurrent updates at 1733164206836 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164206836Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733164206836Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164206837 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164206838 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164206855 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164206855Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164206878 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164206893 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164206893Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164207307 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164207322 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164207322Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164207337 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164207352 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164207352Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23fd128d: reopening flushed file at 1733164207370 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26f4fd7b: reopening flushed file at 1733164207380 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ccfe4a2: reopening flushed file at 1733164207391 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36dfdec3: reopening flushed file at 1733164207402 (+11 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 576ms, sequenceid=29, compaction requested=false at 1733164207411 (+9 ms)Writing region close event to WAL at 1733164207412 (+1 ms)Closed at 1733164207412
2024-12-02T18:30:07,413 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:07,413 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:07,413 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:07,413 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:07,413 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:07,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37793 is added to blk_1073741830_1006 (size=10311)
2024-12-02T18:30:07,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39673 is added to blk_1073741830_1006 (size=10311)
2024-12-02T18:30:07,416 INFO  [M:0;95451c90bed2:46319 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:30:07,416 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:07,416 INFO  [M:0;95451c90bed2:46319 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46319
2024-12-02T18:30:07,416 INFO  [M:0;95451c90bed2:46319 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:07,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:07,585 INFO  [M:0;95451c90bed2:46319 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:07,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46319-0x101974effeb0000, quorum=127.0.0.1:54967, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:07,588 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e721f3d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:07,588 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:07,588 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:07,588 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:07,588 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:07,589 WARN  [BP-1991308746-172.17.0.3-1733164202958 heartbeating to localhost/127.0.0.1:45303 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:07,589 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:07,590 WARN  [BP-1991308746-172.17.0.3-1733164202958 heartbeating to localhost/127.0.0.1:45303 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1991308746-172.17.0.3-1733164202958 (Datanode Uuid 494a9935-39ed-4fb4-a254-0e35f408daeb) service to localhost/127.0.0.1:45303
2024-12-02T18:30:07,590 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:07,590 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data3/current/BP-1991308746-172.17.0.3-1733164202958 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:07,590 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data4/current/BP-1991308746-172.17.0.3-1733164202958 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:07,591 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:07,592 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b9e849{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:07,593 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:07,593 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:07,593 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:07,593 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:07,594 WARN  [BP-1991308746-172.17.0.3-1733164202958 heartbeating to localhost/127.0.0.1:45303 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:07,594 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:07,594 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:07,594 WARN  [BP-1991308746-172.17.0.3-1733164202958 heartbeating to localhost/127.0.0.1:45303 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1991308746-172.17.0.3-1733164202958 (Datanode Uuid 451b7e0f-858f-45e1-b320-7850d33e9cb8) service to localhost/127.0.0.1:45303
2024-12-02T18:30:07,595 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data1/current/BP-1991308746-172.17.0.3-1733164202958 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:07,595 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/cluster_45e972d8-03c7-817e-0478-6c0e3ab99d04/data/data2/current/BP-1991308746-172.17.0.3-1733164202958 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:07,596 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:07,602 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51b7943b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:07,602 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:07,602 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:07,603 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:07,603 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:07,610 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.log.dir so I do NOT create it in target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/50b294cc-7983-03bc-15c5-fd3bbe57c974/hadoop.tmp.dir so I do NOT create it in target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1, deleteOnExit=true
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/test.cache.data in system properties and HBase conf
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:30:07,629 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:30:07,630 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:07,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:30:07,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:30:07,643 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:07,791 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:07,876 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:30:07,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:07,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:07,899 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:07,900 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:08,112 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:08,117 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:08,120 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:08,120 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:08,120 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:08,121 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:08,121 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@692b8c40{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:08,122 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ab5393f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:08,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c461833{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-38965-hadoop-hdfs-3_4_1-tests_jar-_-any-4514116788763069358/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:08,214 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33e53d1d{HTTP/1.1, (http/1.1)}{localhost:38965}
2024-12-02T18:30:08,214 INFO  [Time-limited test {}] server.Server(415): Started @103600ms
2024-12-02T18:30:08,224 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:08,476 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:08,479 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:08,480 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:08,480 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:08,480 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:08,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a4bf55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:08,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23bb5222{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:08,571 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5dd91be7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-34065-hadoop-hdfs-3_4_1-tests_jar-_-any-5681964373501050485/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:08,571 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@55cdd36{HTTP/1.1, (http/1.1)}{localhost:34065}
2024-12-02T18:30:08,571 INFO  [Time-limited test {}] server.Server(415): Started @103957ms
2024-12-02T18:30:08,572 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:08,598 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:08,601 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:08,602 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:08,602 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:08,602 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:08,603 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42443481{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:08,603 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@144f8866{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:08,693 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@24c93faa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-38413-hadoop-hdfs-3_4_1-tests_jar-_-any-17119832760817243181/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:08,694 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e8113bb{HTTP/1.1, (http/1.1)}{localhost:38413}
2024-12-02T18:30:08,694 INFO  [Time-limited test {}] server.Server(415): Started @104080ms
2024-12-02T18:30:08,695 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:09,663 WARN  [Thread-665 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data1/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:09,665 WARN  [Thread-666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data2/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:09,689 WARN  [Thread-629 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:09,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x226c496f04b9d955 with lease ID 0x5c8e6f81af6c580: Processing first storage report for DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60 from datanode DatanodeRegistration(127.0.0.1:40249, datanodeUuid=8db3c922-d24f-447b-854f-a64f0e04fc1f, infoPort=35117, infoSecurePort=0, ipcPort=32931, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:09,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x226c496f04b9d955 with lease ID 0x5c8e6f81af6c580: from storage DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60 node DatanodeRegistration(127.0.0.1:40249, datanodeUuid=8db3c922-d24f-447b-854f-a64f0e04fc1f, infoPort=35117, infoSecurePort=0, ipcPort=32931, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:09,693 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x226c496f04b9d955 with lease ID 0x5c8e6f81af6c580: Processing first storage report for DS-c6db8670-a18d-4dd1-bd62-3112549df454 from datanode DatanodeRegistration(127.0.0.1:40249, datanodeUuid=8db3c922-d24f-447b-854f-a64f0e04fc1f, infoPort=35117, infoSecurePort=0, ipcPort=32931, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:09,693 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x226c496f04b9d955 with lease ID 0x5c8e6f81af6c580: from storage DS-c6db8670-a18d-4dd1-bd62-3112549df454 node DatanodeRegistration(127.0.0.1:40249, datanodeUuid=8db3c922-d24f-447b-854f-a64f0e04fc1f, infoPort=35117, infoSecurePort=0, ipcPort=32931, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:09,805 WARN  [Thread-676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data3/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:09,806 WARN  [Thread-677 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data4/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:09,825 WARN  [Thread-652 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:09,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e9f91c569d4d032 with lease ID 0x5c8e6f81af6c581: Processing first storage report for DS-e89f1db7-47ae-48ed-961d-140e674870de from datanode DatanodeRegistration(127.0.0.1:33679, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=36237, infoSecurePort=0, ipcPort=33457, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:09,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e9f91c569d4d032 with lease ID 0x5c8e6f81af6c581: from storage DS-e89f1db7-47ae-48ed-961d-140e674870de node DatanodeRegistration(127.0.0.1:33679, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=36237, infoSecurePort=0, ipcPort=33457, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:30:09,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e9f91c569d4d032 with lease ID 0x5c8e6f81af6c581: Processing first storage report for DS-63fc2016-b1ae-4261-962a-a3ab68f4e3d3 from datanode DatanodeRegistration(127.0.0.1:33679, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=36237, infoSecurePort=0, ipcPort=33457, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:09,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e9f91c569d4d032 with lease ID 0x5c8e6f81af6c581: from storage DS-63fc2016-b1ae-4261-962a-a3ab68f4e3d3 node DatanodeRegistration(127.0.0.1:33679, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=36237, infoSecurePort=0, ipcPort=33457, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:09,835 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb
2024-12-02T18:30:09,841 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/zookeeper_0, clientPort=56946, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:30:09,842 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56946
2024-12-02T18:30:09,842 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:09,843 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:09,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:09,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:09,853 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df with version=8
2024-12-02T18:30:09,853 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:30:09,855 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:09,856 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43127
2024-12-02T18:30:09,857 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43127 connecting to ZooKeeper ensemble=127.0.0.1:56946
2024-12-02T18:30:09,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431270x0, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:09,914 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43127-0x101974f125a0000 connected
2024-12-02T18:30:09,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:09,999 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:10,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:10,003 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df, hbase.cluster.distributed=false
2024-12-02T18:30:10,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:10,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43127
2024-12-02T18:30:10,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43127
2024-12-02T18:30:10,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43127
2024-12-02T18:30:10,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43127
2024-12-02T18:30:10,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43127
2024-12-02T18:30:10,037 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:10,037 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:30:10,038 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:10,039 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42043
2024-12-02T18:30:10,041 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42043 connecting to ZooKeeper ensemble=127.0.0.1:56946
2024-12-02T18:30:10,042 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:10,044 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:10,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420430x0, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:10,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420430x0, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:10,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42043-0x101974f125a0001 connected
2024-12-02T18:30:10,059 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:30:10,059 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:30:10,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:30:10,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:10,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42043
2024-12-02T18:30:10,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42043
2024-12-02T18:30:10,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42043
2024-12-02T18:30:10,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42043
2024-12-02T18:30:10,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42043
2024-12-02T18:30:10,073 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:43127
2024-12-02T18:30:10,073 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:10,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:10,085 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:10,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,096 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:30:10,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,43127,1733164209855 from backup master directory
2024-12-02T18:30:10,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:10,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:10,105 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:10,105 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,113 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/hbase.id] with ID: e606bebc-94bf-4458-8dad-69bed3bbce45
2024-12-02T18:30:10,113 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/.tmp/hbase.id
2024-12-02T18:30:10,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:10,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:10,122 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/.tmp/hbase.id]:[hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/hbase.id]
2024-12-02T18:30:10,141 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:10,141 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:30:10,144 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-02T18:30:10,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:10,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:10,166 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:30:10,166 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:30:10,167 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:10,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:10,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:10,176 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store
2024-12-02T18:30:10,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:10,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:10,183 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:10,183 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:10,183 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:10,183 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:10,183 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:10,183 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:10,183 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:10,184 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164210183Disabling compacts and flushes for region at 1733164210183Disabling writes for close at 1733164210183Writing region close event to WAL at 1733164210183Closed at 1733164210183
2024-12-02T18:30:10,185 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/.initializing
2024-12-02T18:30:10,185 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,187 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C43127%2C1733164209855, suffix=, logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855, archiveDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/oldWALs, maxLogs=10
2024-12-02T18:30:10,188 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43127%2C1733164209855.1733164210188
2024-12-02T18:30:10,192 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188
2024-12-02T18:30:10,193 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117),(127.0.0.1/127.0.0.1:36237:36237)]
2024-12-02T18:30:10,194 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:10,194 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:10,194 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,194 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,196 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:10,197 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:30:10,197 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,197 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:10,197 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:10,199 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:30:10,199 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,199 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:10,199 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:10,200 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:30:10,200 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:10,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:10,202 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:30:10,202 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,203 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:10,203 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,204 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,204 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,206 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,206 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,206 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:30:10,208 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:10,210 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:10,211 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=786265, jitterRate=-2.1286308765411377E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:30:10,212 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164210194Initializing all the Stores at 1733164210195 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164210195Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164210195Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164210195Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164210195Cleaning up temporary data from old regions at 1733164210206 (+11 ms)Region opened successfully at 1733164210212 (+6 ms)
2024-12-02T18:30:10,212 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:30:10,216 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c3e2c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:10,217 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:30:10,217 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:30:10,218 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:30:10,218 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:30:10,218 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:30:10,219 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:30:10,219 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:30:10,221 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:30:10,222 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:30:10,232 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:30:10,232 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:30:10,233 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:30:10,242 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:30:10,243 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:30:10,244 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:30:10,253 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:30:10,254 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:30:10,263 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:30:10,266 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:30:10,274 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:30:10,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:10,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:10,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,285 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,43127,1733164209855, sessionid=0x101974f125a0000, setting cluster-up flag (Was=false)
2024-12-02T18:30:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,337 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:30:10,339 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,390 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:30:10,393 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,43127,1733164209855
2024-12-02T18:30:10,396 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:30:10,399 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:10,400 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:30:10,400 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:30:10,400 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,43127,1733164209855
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:10,403 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164240404
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:30:10,404 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:30:10,405 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:30:10,405 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:30:10,405 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:30:10,406 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164210406,5,FailOnTimeoutGroup]
2024-12-02T18:30:10,406 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164210406,5,FailOnTimeoutGroup]
2024-12-02T18:30:10,406 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,406 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:30:10,406 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,406 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,406 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,406 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:30:10,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:10,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:10,413 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:30:10,414 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df
2024-12-02T18:30:10,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:10,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:10,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:10,423 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:30:10,425 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:30:10,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,425 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:10,425 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:30:10,427 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:30:10,427 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,427 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:10,428 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:30:10,429 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:30:10,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,430 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:10,430 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:30:10,431 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:30:10,431 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:10,432 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:10,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:30:10,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740
2024-12-02T18:30:10,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740
2024-12-02T18:30:10,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:30:10,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:30:10,435 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:30:10,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:30:10,438 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:10,439 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809572, jitterRate=0.029425084590911865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:30:10,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164210421Initializing all the Stores at 1733164210423 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164210423Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164210423Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164210423Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164210423Cleaning up temporary data from old regions at 1733164210434 (+11 ms)Region opened successfully at 1733164210440 (+6 ms)
2024-12-02T18:30:10,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:30:10,440 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:30:10,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:30:10,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:30:10,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:30:10,441 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:10,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164210440Disabling compacts and flushes for region at 1733164210440Disabling writes for close at 1733164210441 (+1 ms)Writing region close event to WAL at 1733164210441Closed at 1733164210441
2024-12-02T18:30:10,443 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:10,443 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:30:10,443 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:30:10,444 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:30:10,445 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:30:10,465 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(746): ClusterId : e606bebc-94bf-4458-8dad-69bed3bbce45
2024-12-02T18:30:10,465 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:30:10,476 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:30:10,476 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:30:10,486 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:30:10,486 DEBUG [RS:0;95451c90bed2:42043 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b749963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:10,499 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:42043
2024-12-02T18:30:10,499 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:30:10,499 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:30:10,499 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:30:10,500 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,43127,1733164209855 with port=42043, startcode=1733164210037
2024-12-02T18:30:10,500 DEBUG [RS:0;95451c90bed2:42043 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:30:10,502 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34519, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:30:10,503 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43127 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,503 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43127 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,505 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df
2024-12-02T18:30:10,505 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40269
2024-12-02T18:30:10,505 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:30:10,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:10,516 DEBUG [RS:0;95451c90bed2:42043 {}] zookeeper.ZKUtil(111): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,516 WARN  [RS:0;95451c90bed2:42043 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:10,517 INFO  [RS:0;95451c90bed2:42043 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:10,517 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,517 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,42043,1733164210037]
2024-12-02T18:30:10,521 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:30:10,523 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:30:10,524 INFO  [RS:0;95451c90bed2:42043 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:30:10,524 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,524 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:30:10,525 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:30:10,525 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,525 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,525 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,525 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,525 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:10,526 DEBUG [RS:0;95451c90bed2:42043 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,527 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,42043,1733164210037-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:30:10,544 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:30:10,544 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,42043,1733164210037-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,545 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,545 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.Replication(171): 95451c90bed2,42043,1733164210037 started
2024-12-02T18:30:10,556 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:10,556 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,42043,1733164210037, RpcServer on 95451c90bed2/172.17.0.3:42043, sessionid=0x101974f125a0001
2024-12-02T18:30:10,556 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:30:10,556 DEBUG [RS:0;95451c90bed2:42043 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,556 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,42043,1733164210037'
2024-12-02T18:30:10,556 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:30:10,557 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:30:10,558 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:30:10,558 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:30:10,558 DEBUG [RS:0;95451c90bed2:42043 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,558 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,42043,1733164210037'
2024-12-02T18:30:10,558 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:30:10,559 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:30:10,559 DEBUG [RS:0;95451c90bed2:42043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:30:10,559 INFO  [RS:0;95451c90bed2:42043 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:30:10,559 INFO  [RS:0;95451c90bed2:42043 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:30:10,595 WARN  [95451c90bed2:43127 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:30:10,664 INFO  [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C42043%2C1733164210037, suffix=, logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037, archiveDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs, maxLogs=32
2024-12-02T18:30:10,667 INFO  [RS:0;95451c90bed2:42043 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164210667
2024-12-02T18:30:10,675 INFO  [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667
2024-12-02T18:30:10,680 DEBUG [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117),(127.0.0.1/127.0.0.1:36237:36237)]
2024-12-02T18:30:10,846 DEBUG [95451c90bed2:43127 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:30:10,847 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,42043,1733164210037
2024-12-02T18:30:10,851 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,42043,1733164210037, state=OPENING
2024-12-02T18:30:10,906 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:30:10,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:10,917 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:30:10,917 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,42043,1733164210037}]
2024-12-02T18:30:10,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:10,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:11,072 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:30:11,074 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45281, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:30:11,078 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:30:11,078 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:11,080 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C42043%2C1733164210037.meta, suffix=.meta, logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037, archiveDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs, maxLogs=32
2024-12-02T18:30:11,081 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
2024-12-02T18:30:11,087 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
2024-12-02T18:30:11,112 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:35117:35117)]
2024-12-02T18:30:11,117 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:11,117 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:30:11,117 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:30:11,117 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:30:11,117 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:30:11,117 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:11,118 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:30:11,118 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:30:11,124 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:30:11,126 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:30:11,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,126 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:11,127 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:30:11,127 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:30:11,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,128 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:11,128 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:30:11,129 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:30:11,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:11,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:30:11,131 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:30:11,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,131 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:11,131 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:30:11,132 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740
2024-12-02T18:30:11,133 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740
2024-12-02T18:30:11,134 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:30:11,134 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:30:11,135 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:30:11,136 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:30:11,137 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=806589, jitterRate=0.025631383061408997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:30:11,137 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:30:11,138 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164211118Writing region info on filesystem at 1733164211118Initializing all the Stores at 1733164211119 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164211119Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164211124 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164211124Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164211124Cleaning up temporary data from old regions at 1733164211134 (+10 ms)Running coprocessor post-open hooks at 1733164211137 (+3 ms)Region opened successfully at 1733164211137
2024-12-02T18:30:11,139 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164211072
2024-12-02T18:30:11,142 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:30:11,142 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:30:11,143 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,42043,1733164210037
2024-12-02T18:30:11,144 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,42043,1733164210037, state=OPEN
2024-12-02T18:30:11,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:30:11,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:30:11,178 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,42043,1733164210037
2024-12-02T18:30:11,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:11,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:30:11,182 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:30:11,182 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,42043,1733164210037 in 261 msec
2024-12-02T18:30:11,186 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:30:11,186 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec
2024-12-02T18:30:11,187 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:11,187 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:30:11,189 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:30:11,189 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,42043,1733164210037, seqNum=-1]
2024-12-02T18:30:11,189 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:30:11,190 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:30:11,197 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 797 msec
2024-12-02T18:30:11,197 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164211197, completionTime=-1
2024-12-02T18:30:11,197 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:30:11,197 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:30:11,199 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:30:11,199 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164271199
2024-12-02T18:30:11,199 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164331199
2024-12-02T18:30:11,199 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:43127, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,200 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,202 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:30:11,204 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.099sec
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:30:11,205 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:30:11,208 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:30:11,208 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:30:11,208 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43127,1733164209855-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142e8458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:11,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,43127,-1 for getting cluster id
2024-12-02T18:30:11,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:30:11,268 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e606bebc-94bf-4458-8dad-69bed3bbce45'
2024-12-02T18:30:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:30:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e606bebc-94bf-4458-8dad-69bed3bbce45"

2024-12-02T18:30:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b8bda3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,43127,-1]
2024-12-02T18:30:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:30:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:11,272 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:30:11,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee117d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:30:11,273 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:30:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,42043,1733164210037, seqNum=-1]
2024-12-02T18:30:11,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:30:11,278 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:30:11,281 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,43127,1733164209855
2024-12-02T18:30:11,281 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:11,286 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:11,301 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:30:11,302 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:11,302 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37951
2024-12-02T18:30:11,303 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37951 connecting to ZooKeeper ensemble=127.0.0.1:56946
2024-12-02T18:30:11,304 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:11,305 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:11,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379510x0, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:11,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37951-0x101974f125a0002 connected
2024-12-02T18:30:11,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-02T18:30:11,327 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-02T18:30:11,328 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:30:11,329 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:30:11,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:30:11,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:11,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37951
2024-12-02T18:30:11,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37951
2024-12-02T18:30:11,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37951
2024-12-02T18:30:11,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37951
2024-12-02T18:30:11,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37951
2024-12-02T18:30:11,338 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(746): ClusterId : e606bebc-94bf-4458-8dad-69bed3bbce45
2024-12-02T18:30:11,338 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:30:11,349 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:30:11,349 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:30:11,360 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:30:11,360 DEBUG [RS:1;95451c90bed2:37951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c823077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:11,378 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;95451c90bed2:37951
2024-12-02T18:30:11,378 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:30:11,378 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:30:11,378 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:30:11,379 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,43127,1733164209855 with port=37951, startcode=1733164211301
2024-12-02T18:30:11,379 DEBUG [RS:1;95451c90bed2:37951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:30:11,380 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51493, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:30:11,381 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43127 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,381 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43127 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,383 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df
2024-12-02T18:30:11,383 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40269
2024-12-02T18:30:11,383 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:30:11,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:11,395 DEBUG [RS:1;95451c90bed2:37951 {}] zookeeper.ZKUtil(111): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,395 WARN  [RS:1;95451c90bed2:37951 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:11,395 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,37951,1733164211301]
2024-12-02T18:30:11,395 INFO  [RS:1;95451c90bed2:37951 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:11,395 DEBUG [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,399 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:30:11,401 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:30:11,402 INFO  [RS:1;95451c90bed2:37951 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:30:11,402 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,402 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:30:11,403 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:30:11,403 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,403 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:11,404 DEBUG [RS:1;95451c90bed2:37951 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:30:11,405 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,405 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,405 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,405 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,405 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,406 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,37951,1733164211301-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:30:11,420 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:30:11,420 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,37951,1733164211301-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,420 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,420 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.Replication(171): 95451c90bed2,37951,1733164211301 started
2024-12-02T18:30:11,431 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:11,432 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,37951,1733164211301, RpcServer on 95451c90bed2/172.17.0.3:37951, sessionid=0x101974f125a0002
2024-12-02T18:30:11,432 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;95451c90bed2:37951,5,FailOnTimeoutGroup]
2024-12-02T18:30:11,432 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:30:11,432 DEBUG [RS:1;95451c90bed2:37951 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,432 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,37951,1733164211301'
2024-12-02T18:30:11,432 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:30:11,432 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-02T18:30:11,433 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,37951,1733164211301
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,37951,1733164211301'
2024-12-02T18:30:11,433 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:30:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 95451c90bed2,43127,1733164209855
2024-12-02T18:30:11,434 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:30:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2054a2d1
2024-12-02T18:30:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-02T18:30:11,434 DEBUG [RS:1;95451c90bed2:37951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:30:11,434 INFO  [RS:1;95451c90bed2:37951 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:30:11,434 INFO  [RS:1;95451c90bed2:37951 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:30:11,436 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-02T18:30:11,436 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-02T18:30:11,436 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-02T18:30:11,437 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:30:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-02T18:30:11,439 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-02T18:30:11,439 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,439 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-02T18:30:11,440 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-02T18:30:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:30:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741835_1011 (size=393)
2024-12-02T18:30:11,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741835_1011 (size=393)
2024-12-02T18:30:11,448 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f395c8610517974fa838bf7ff6019549, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df
2024-12-02T18:30:11,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40249 is added to blk_1073741836_1012 (size=76)
2024-12-02T18:30:11,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33679 is added to blk_1073741836_1012 (size=76)
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing f395c8610517974fa838bf7ff6019549, disabling compactions & flushes
2024-12-02T18:30:11,455 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549. after waiting 0 ms
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,455 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,455 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for f395c8610517974fa838bf7ff6019549:
Waiting for close lock at 1733164211455Disabling compacts and flushes for region at 1733164211455Disabling writes for close at 1733164211455Writing region close event to WAL at 1733164211455Closed at 1733164211455
2024-12-02T18:30:11,457 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-02T18:30:11,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733164211457"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164211457"}]},"ts":"1733164211457"}
2024-12-02T18:30:11,460 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-02T18:30:11,461 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-02T18:30:11,461 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164211461"}]},"ts":"1733164211461"}
2024-12-02T18:30:11,464 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-02T18:30:11,464 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f395c8610517974fa838bf7ff6019549, ASSIGN}]
2024-12-02T18:30:11,466 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f395c8610517974fa838bf7ff6019549, ASSIGN
2024-12-02T18:30:11,467 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f395c8610517974fa838bf7ff6019549, ASSIGN; state=OFFLINE, location=95451c90bed2,42043,1733164210037; forceNewPlan=false, retain=false
2024-12-02T18:30:11,537 INFO  [RS:1;95451c90bed2:37951 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C37951%2C1733164211301, suffix=, logDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301, archiveDir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs, maxLogs=32
2024-12-02T18:30:11,537 INFO  [RS:1;95451c90bed2:37951 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C37951%2C1733164211301.1733164211537
2024-12-02T18:30:11,543 INFO  [RS:1;95451c90bed2:37951 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
2024-12-02T18:30:11,544 DEBUG [RS:1;95451c90bed2:37951 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35117:35117),(127.0.0.1/127.0.0.1:36237:36237)]
2024-12-02T18:30:11,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-02T18:30:11,618 INFO  [95451c90bed2:43127 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-02T18:30:11,619 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f395c8610517974fa838bf7ff6019549, regionState=OPENING, regionLocation=95451c90bed2,42043,1733164210037
2024-12-02T18:30:11,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f395c8610517974fa838bf7ff6019549, ASSIGN because future has completed
2024-12-02T18:30:11,624 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f395c8610517974fa838bf7ff6019549, server=95451c90bed2,42043,1733164210037}]
2024-12-02T18:30:11,788 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,789 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f395c8610517974fa838bf7ff6019549, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:11,790 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,790 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:11,790 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,790 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,792 INFO  [StoreOpener-f395c8610517974fa838bf7ff6019549-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f395c8610517974fa838bf7ff6019549 
2024-12-02T18:30:11,793 INFO  [StoreOpener-f395c8610517974fa838bf7ff6019549-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f395c8610517974fa838bf7ff6019549 columnFamilyName info
2024-12-02T18:30:11,793 DEBUG [StoreOpener-f395c8610517974fa838bf7ff6019549-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:11,794 INFO  [StoreOpener-f395c8610517974fa838bf7ff6019549-1 {}] regionserver.HStore(327): Store=f395c8610517974fa838bf7ff6019549/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:11,794 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,795 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,795 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,796 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,796 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,798 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,800 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:11,800 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f395c8610517974fa838bf7ff6019549; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715108, jitterRate=-0.09069375693798065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:30:11,800 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:11,801 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f395c8610517974fa838bf7ff6019549:
Running coprocessor pre-open hook at 1733164211790Writing region info on filesystem at 1733164211790Initializing all the Stores at 1733164211791 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164211791Cleaning up temporary data from old regions at 1733164211796 (+5 ms)Running coprocessor post-open hooks at 1733164211800 (+4 ms)Region opened successfully at 1733164211801 (+1 ms)
2024-12-02T18:30:11,802 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549., pid=6, masterSystemTime=1733164211778
2024-12-02T18:30:11,804 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,804 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:11,805 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f395c8610517974fa838bf7ff6019549, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,42043,1733164210037
2024-12-02T18:30:11,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f395c8610517974fa838bf7ff6019549, server=95451c90bed2,42043,1733164210037 because future has completed
2024-12-02T18:30:11,811 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-02T18:30:11,812 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f395c8610517974fa838bf7ff6019549, server=95451c90bed2,42043,1733164210037 in 184 msec
2024-12-02T18:30:11,815 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-02T18:30:11,815 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f395c8610517974fa838bf7ff6019549, ASSIGN in 348 msec
2024-12-02T18:30:11,817 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-02T18:30:11,817 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164211817"}]},"ts":"1733164211817"}
2024-12-02T18:30:11,821 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-02T18:30:11,823 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-02T18:30:11,826 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 386 msec
2024-12-02T18:30:16,620 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:30:16,622 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:16,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:16,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:16,652 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:16,663 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-02T18:30:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43127 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:30:21,546 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-02T18:30:21,546 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-02T18:30:21,550 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-02T18:30:21,550 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:21,570 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:21,574 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:21,575 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:21,575 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:21,575 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:30:21,576 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fb911ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:21,576 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@719bbb9b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:21,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-02T18:30:21,591 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-02T18:30:21,716 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2f0a2519{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-36353-hadoop-hdfs-3_4_1-tests_jar-_-any-15169194208855160710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:21,716 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68004957{HTTP/1.1, (http/1.1)}{localhost:36353}
2024-12-02T18:30:21,716 INFO  [Time-limited test {}] server.Server(415): Started @117103ms
2024-12-02T18:30:21,718 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:21,799 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:21,803 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:21,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:21,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:21,804 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:21,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64b86931{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:21,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@624ef820{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:21,915 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@786c904f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-41647-hadoop-hdfs-3_4_1-tests_jar-_-any-10485673824977583378/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:21,916 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@18a0c61c{HTTP/1.1, (http/1.1)}{localhost:41647}
2024-12-02T18:30:21,916 INFO  [Time-limited test {}] server.Server(415): Started @117302ms
2024-12-02T18:30:21,917 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:21,951 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:21,953 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:21,955 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:21,955 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:21,955 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:30:21,956 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@339da5f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:21,957 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fb4bc9e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:22,088 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a9b13ea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-35053-hadoop-hdfs-3_4_1-tests_jar-_-any-14926351947280856295/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:22,089 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@61f29988{HTTP/1.1, (http/1.1)}{localhost:35053}
2024-12-02T18:30:22,089 INFO  [Time-limited test {}] server.Server(415): Started @117475ms
2024-12-02T18:30:22,090 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:22,934 WARN  [Thread-860 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:22,934 WARN  [Thread-859 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:22,951 WARN  [Thread-801 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:22,954 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5bf49454c475ddec with lease ID 0x5c8e6f81af6c582: Processing first storage report for DS-f0870539-7f8d-4124-8e24-f89d356151d6 from datanode DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:22,954 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5bf49454c475ddec with lease ID 0x5c8e6f81af6c582: from storage DS-f0870539-7f8d-4124-8e24-f89d356151d6 node DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:30:22,954 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5bf49454c475ddec with lease ID 0x5c8e6f81af6c582: Processing first storage report for DS-ac0c5b9d-e0ea-4d68-9045-e022c1152c0f from datanode DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:22,954 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5bf49454c475ddec with lease ID 0x5c8e6f81af6c582: from storage DS-ac0c5b9d-e0ea-4d68-9045-e022c1152c0f node DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:23,276 WARN  [Thread-871 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data7/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:23,276 WARN  [Thread-872 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data8/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:23,294 WARN  [Thread-823 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:23,296 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf64ce4ceae85311 with lease ID 0x5c8e6f81af6c583: Processing first storage report for DS-24b27e8b-9ef7-42d8-a981-3faf1236491b from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=4601ea68-ddbc-421a-b898-89868fc27e3a, infoPort=36773, infoSecurePort=0, ipcPort=39465, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:23,296 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf64ce4ceae85311 with lease ID 0x5c8e6f81af6c583: from storage DS-24b27e8b-9ef7-42d8-a981-3faf1236491b node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=4601ea68-ddbc-421a-b898-89868fc27e3a, infoPort=36773, infoSecurePort=0, ipcPort=39465, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:23,296 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf64ce4ceae85311 with lease ID 0x5c8e6f81af6c583: Processing first storage report for DS-97dd5beb-21f1-485e-9de2-b18f014643d1 from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=4601ea68-ddbc-421a-b898-89868fc27e3a, infoPort=36773, infoSecurePort=0, ipcPort=39465, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:23,296 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf64ce4ceae85311 with lease ID 0x5c8e6f81af6c583: from storage DS-97dd5beb-21f1-485e-9de2-b18f014643d1 node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=4601ea68-ddbc-421a-b898-89868fc27e3a, infoPort=36773, infoSecurePort=0, ipcPort=39465, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:23,374 WARN  [Thread-883 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data10/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:23,374 WARN  [Thread-882 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data9/current/BP-209545290-172.17.0.3-1733164207654/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:23,397 WARN  [Thread-845 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:23,400 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x390721e61b1200e0 with lease ID 0x5c8e6f81af6c584: Processing first storage report for DS-70407dbe-8205-4621-ba2e-2e1699c87aac from datanode DatanodeRegistration(127.0.0.1:42381, datanodeUuid=64a40d03-e7d2-456f-b10e-46e155b2cd51, infoPort=41281, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:23,400 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x390721e61b1200e0 with lease ID 0x5c8e6f81af6c584: from storage DS-70407dbe-8205-4621-ba2e-2e1699c87aac node DatanodeRegistration(127.0.0.1:42381, datanodeUuid=64a40d03-e7d2-456f-b10e-46e155b2cd51, infoPort=41281, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:30:23,400 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x390721e61b1200e0 with lease ID 0x5c8e6f81af6c584: Processing first storage report for DS-7f5f41ac-2bab-4d58-8020-33dcd88e50b4 from datanode DatanodeRegistration(127.0.0.1:42381, datanodeUuid=64a40d03-e7d2-456f-b10e-46e155b2cd51, infoPort=41281, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654)
2024-12-02T18:30:23,400 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x390721e61b1200e0 with lease ID 0x5c8e6f81af6c584: from storage DS-7f5f41ac-2bab-4d58-8020-33dcd88e50b4 node DatanodeRegistration(127.0.0.1:42381, datanodeUuid=64a40d03-e7d2-456f-b10e-46e155b2cd51, infoPort=41281, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:23,433 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,433 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,433 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,434 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 block BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:23,434 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 block BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:23,434 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta block BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:23,434 WARN  [PacketResponder: BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33679] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,434 WARN  [PacketResponder: BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33679] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,435 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,436 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 block BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:23,436 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:57646 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57646 dst: /127.0.0.1:40249
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,436 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-45597091_22 at /127.0.0.1:40772 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33679:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40772 dst: /127.0.0.1:33679
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,435 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-45597091_22 at /127.0.0.1:57622 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57622 dst: /127.0.0.1:40249
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,437 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:40786 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33679:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40786 dst: /127.0.0.1:33679
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,438 WARN  [PacketResponder: BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33679] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-631421306_22 at /127.0.0.1:57688 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:40249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57688 dst: /127.0.0.1:40249
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,437 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:57652 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40249:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57652 dst: /127.0.0.1:40249
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-631421306_22 at /127.0.0.1:40830 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:33679:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40830 dst: /127.0.0.1:33679
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,436 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:40798 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33679:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40798 dst: /127.0.0.1:33679
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@24c93faa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:23,446 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e8113bb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:23,446 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:23,446 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@144f8866{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:23,446 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42443481{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:23,447 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:23,447 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:23,447 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid d1e32ab3-9f16-4281-a5f9-533d4a7d7e90) service to localhost/127.0.0.1:40269
2024-12-02T18:30:23,447 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:23,448 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data3/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:23,449 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data4/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:23,449 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:23,457 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 block BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,457 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta block BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,460 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@6f149901 {}] datanode.DataXceiver(331): 127.0.0.1:40249:DataXceiver error processing unknown operation  src: /127.0.0.1:53080 dst: /127.0.0.1:40249
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:23,461 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 block BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,485 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 block BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,486 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5dd91be7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:23,487 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@55cdd36{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:23,487 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:23,487 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23bb5222{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:23,487 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a4bf55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:23,489 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:23,489 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:23,489 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid 8db3c922-d24f-447b-854f-a64f0e04fc1f) service to localhost/127.0.0.1:40269
2024-12-02T18:30:23,489 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:23,489 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data1/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:23,490 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data2/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:23,490 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549., hostname=95451c90bed2,42043,1733164210037, seqNum=2]
2024-12-02T18:30:23,496 ERROR [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df-prefix:95451c90bed2,42043,1733164210037 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,496 WARN  [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df-prefix:95451c90bed2,42043,1733164210037 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,496 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C42043%2C1733164210037:(num 1733164210667) roll requested
2024-12-02T18:30:23,496 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164223496
2024-12-02T18:30:23,513 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:23,514 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:23,514 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:23,514 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:23,514 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:23,514 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496
2024-12-02T18:30:23,514 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,515 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:23,516 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-02T18:30:23,516 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-02T18:30:23,516 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667
2024-12-02T18:30:23,520 WARN  [IPC Server handler 2 on default port 40269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-12-02T18:30:23,524 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 after 5ms
2024-12-02T18:30:23,524 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41281:41281),(127.0.0.1/127.0.0.1:36773:36773)]
2024-12-02T18:30:23,524 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:24,168 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:25,406 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:25,525 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:25,526 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496
2024-12-02T18:30:25,526 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:25,527 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496 block BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:25,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:42080 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:42381:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42080 dst: /127.0.0.1:42381
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:25,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:38390 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41211:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38390 dst: /127.0.0.1:41211
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:25,592 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a9b13ea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:25,593 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@61f29988{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:25,593 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:25,593 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fb4bc9e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:25,593 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@339da5f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:25,595 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:25,595 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:25,595 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid 64a40d03-e7d2-456f-b10e-46e155b2cd51) service to localhost/127.0.0.1:40269
2024-12-02T18:30:25,595 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:25,595 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data9/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:25,595 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data10/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:25,596 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:26,168 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:27,406 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:27,525 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 after 4008ms
2024-12-02T18:30:27,525 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:27,526 WARN  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]
2024-12-02T18:30:27,526 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C42043%2C1733164210037:(num 1733164223496) roll requested
2024-12-02T18:30:27,526 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164227526
2024-12-02T18:30:27,529 WARN  [Thread-903 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:27,529 WARN  [Thread-903 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:27,529 WARN  [Thread-903 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741839_1021
2024-12-02T18:30:27,531 WARN  [Thread-903 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:27,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:27,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:27,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741840_1022
2024-12-02T18:30:27,535 WARN  [Thread-903 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:27,540 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:27,540 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:27,540 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:27,540 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:27,540 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:27,541 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164227526
2024-12-02T18:30:27,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741838_1020 (size=2431)
2024-12-02T18:30:27,543 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42809:42809),(127.0.0.1/127.0.0.1:36773:36773)]
2024-12-02T18:30:27,543 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:27,544 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496 is not closed yet, will try archiving it next time
2024-12-02T18:30:27,601 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-02T18:30:27,944 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:28,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,407 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,544 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,605 WARN  [ResponseProcessor for block BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023
java.io.IOException: Bad response ERROR for BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023 from datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,606 WARN  [DataStreamer for file /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164227526 block BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:29,606 WARN  [PacketResponder: BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41211] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:29,606 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51816 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51816 dst: /127.0.0.1:45691
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:29,607 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:38416 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:41211:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38416 dst: /127.0.0.1:41211
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:29,704 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@786c904f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:29,705 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@18a0c61c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:29,705 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:29,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@624ef820{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:29,706 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64b86931{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:29,707 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:29,707 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid 4601ea68-ddbc-421a-b898-89868fc27e3a) service to localhost/127.0.0.1:40269
2024-12-02T18:30:29,708 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:29,708 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:29,708 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data7/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:29,709 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data8/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:29,709 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:29,720 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:30:29,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/3a5a2a0f4ef04bcfbf0695e0c586c580 is 1080, key is row0002/info:/1733164225597/Put/seqid=0
2024-12-02T18:30:29,740 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,741 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:29,741 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741842_1025
2024-12-02T18:30:29,741 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:29,743 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,743 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:29,743 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741843_1026
2024-12-02T18:30:29,743 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:29,745 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,745 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:29,745 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741844_1027
2024-12-02T18:30:29,745 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:29,747 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:29,747 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51840 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028 to mirror 127.0.0.1:41211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:29,748 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:29,748 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028
2024-12-02T18:30:29,748 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51840 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:29,748 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51840 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51840 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:29,749 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:29,749 WARN  [IPC Server handler 1 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:29,749 WARN  [IPC Server handler 1 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:29,749 WARN  [IPC Server handler 1 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:29,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741846_1029 (size=10347)
2024-12-02T18:30:30,153 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/3a5a2a0f4ef04bcfbf0695e0c586c580
2024-12-02T18:30:30,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/3a5a2a0f4ef04bcfbf0695e0c586c580 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580
2024-12-02T18:30:30,168 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580, entries=5, sequenceid=11, filesize=10.1 K
2024-12-02T18:30:30,169 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for f395c8610517974fa838bf7ff6019549 in 449ms, sequenceid=11, compaction requested=false
2024-12-02T18:30:30,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:30,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:30,345 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-02T18:30:30,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/d91374a3a3e94d3ba55947f8c9153166 is 1080, key is row0007/info:/1733164229722/Put/seqid=0
2024-12-02T18:30:30,351 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:30,351 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:30,352 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741847_1030
2024-12-02T18:30:30,352 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:30,353 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:30,354 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:30,354 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741848_1031
2024-12-02T18:30:30,354 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:30,356 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33679
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:30,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51866 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032 to mirror 127.0.0.1:33679
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:30,357 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:30,357 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032
2024-12-02T18:30:30,357 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51866 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:30,357 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51866 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51866 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:30,357 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:30,360 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:30,360 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51882 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033 to mirror 127.0.0.1:41211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:30,360 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:30,360 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033
2024-12-02T18:30:30,360 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51882 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:30,360 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51882 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51882 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:30,361 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:30,362 WARN  [IPC Server handler 0 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:30,362 WARN  [IPC Server handler 0 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:30,362 WARN  [IPC Server handler 0 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:30,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741851_1034 (size=12506)
2024-12-02T18:30:30,766 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/d91374a3a3e94d3ba55947f8c9153166
2024-12-02T18:30:30,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/d91374a3a3e94d3ba55947f8c9153166 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166
2024-12-02T18:30:30,780 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166, entries=7, sequenceid=24, filesize=12.2 K
2024-12-02T18:30:30,781 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for f395c8610517974fa838bf7ff6019549 in 436ms, sequenceid=24, compaction requested=false
2024-12-02T18:30:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-02T18:30:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166 because midkey is the same as first or last row
2024-12-02T18:30:31,407 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,544 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,544 WARN  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]
2024-12-02T18:30:31,545 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C42043%2C1733164210037:(num 1733164227526) roll requested
2024-12-02T18:30:31,545 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164231545
2024-12-02T18:30:31,548 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,548 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:31,548 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741852_1035
2024-12-02T18:30:31,549 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:31,550 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,550 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:31,550 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741853_1036
2024-12-02T18:30:31,550 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:31,552 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,552 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:31,552 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741854_1037
2024-12-02T18:30:31,553 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:31,556 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,556 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51896 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038 to mirror 127.0.0.1:41211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:31,556 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:31,557 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038
2024-12-02T18:30:31,557 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51896 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-02T18:30:31,557 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51896 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51896 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:31,557 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:31,558 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:31,558 WARN  [IPC Server handler 2 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:31,558 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:31,562 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:31,562 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:31,562 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:31,562 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:31,562 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:31,563 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164227526 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164231545
2024-12-02T18:30:31,563 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42809:42809)]
2024-12-02T18:30:31,564 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:31,564 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164227526 is not closed yet, will try archiving it next time
2024-12-02T18:30:31,564 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164223496 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C42043%2C1733164210037.1733164223496
2024-12-02T18:30:31,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741841_1024 (size=25992)
2024-12-02T18:30:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:31,765 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-02T18:30:31,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9f1e7e43c7a84c97a46dc698efc5a4c4 is 1079, key is tmprow/info:/1733164231764/Put/seqid=0
2024-12-02T18:30:31,772 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,772 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:31,772 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741857_1040
2024-12-02T18:30:31,773 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:31,774 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,775 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:31,775 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741858_1041
2024-12-02T18:30:31,775 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:31,777 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,777 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51914 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042 to mirror 127.0.0.1:41211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:31,777 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:31,778 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042
2024-12-02T18:30:31,778 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51914 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:31,778 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:51914 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51914 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:31,778 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:31,780 WARN  [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:31,780 WARN  [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:31,780 WARN  [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741860_1043
2024-12-02T18:30:31,780 WARN  [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:31,781 WARN  [IPC Server handler 3 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:31,781 WARN  [IPC Server handler 3 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:31,781 WARN  [IPC Server handler 3 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:31,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741861_1044 (size=6027)
2024-12-02T18:30:31,965 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:32,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:32,185 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9f1e7e43c7a84c97a46dc698efc5a4c4
2024-12-02T18:30:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9f1e7e43c7a84c97a46dc698efc5a4c4 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4
2024-12-02T18:30:32,202 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4, entries=1, sequenceid=34, filesize=5.9 K
2024-12-02T18:30:32,203 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f395c8610517974fa838bf7ff6019549 in 438ms, sequenceid=34, compaction requested=true
2024-12-02T18:30:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-02T18:30:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:32,204 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166 because midkey is the same as first or last row
2024-12-02T18:30:32,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f395c8610517974fa838bf7ff6019549:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:30:32,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:30:32,204 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:30:32,206 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:30:32,206 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1541): f395c8610517974fa838bf7ff6019549/info is initiating minor compaction (all files)
2024-12-02T18:30:32,206 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f395c8610517974fa838bf7ff6019549/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:32,206 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4] into tmpdir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp, totalSize=28.2 K
2024-12-02T18:30:32,207 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5a2a0f4ef04bcfbf0695e0c586c580, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733164225597
2024-12-02T18:30:32,207 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91374a3a3e94d3ba55947f8c9153166, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733164229722
2024-12-02T18:30:32,208 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f1e7e43c7a84c97a46dc698efc5a4c4, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733164231764
2024-12-02T18:30:32,227 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f395c8610517974fa838bf7ff6019549#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:30:32,228 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/689597003a474bb59ac6231f967fc0f5 is 1080, key is row0002/info:/1733164225597/Put/seqid=0
2024-12-02T18:30:32,230 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:32,230 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:32,230 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741862_1045
2024-12-02T18:30:32,231 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:32,233 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33679
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:32,233 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53482 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046 to mirror 127.0.0.1:33679
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:32,233 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:32,233 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046
2024-12-02T18:30:32,234 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53482 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:32,234 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53482 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53482 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:32,234 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:32,235 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:32,235 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:32,236 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741864_1047
2024-12-02T18:30:32,236 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:32,238 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42381
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:32,238 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53490 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048 to mirror 127.0.0.1:42381
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:32,239 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:32,239 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048
2024-12-02T18:30:32,239 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53490 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:32,239 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53490 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53490 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:32,239 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:32,240 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:32,240 WARN  [IPC Server handler 2 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:32,240 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:32,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741866_1049 (size=17994)
2024-12-02T18:30:32,651 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/689597003a474bb59ac6231f967fc0f5 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5
2024-12-02T18:30:32,659 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f395c8610517974fa838bf7ff6019549/info of f395c8610517974fa838bf7ff6019549 into 689597003a474bb59ac6231f967fc0f5(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:32,660 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549., storeName=f395c8610517974fa838bf7ff6019549/info, priority=13, startTime=1733164232204; duration=0sec
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 because midkey is the same as first or last row
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 because midkey is the same as first or last row
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 because midkey is the same as first or last row
2024-12-02T18:30:32,660 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:30:32,661 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f395c8610517974fa838bf7ff6019549:info
2024-12-02T18:30:32,968 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741846_1029 to 127.0.0.1:40249 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:32,968 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@63aed152[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741851_1034 to 127.0.0.1:40249 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:33,185 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-02T18:30:33,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a982b80830094c528567738edbcd1642 is 1079, key is tmprow/info:/1733164233183/Put/seqid=0
2024-12-02T18:30:33,192 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42381
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,192 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53502 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050 to mirror 127.0.0.1:42381
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,192 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:33,192 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53502 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050] {}] datanode.BlockReceiver(316): Block 1073741867 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:33,192 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050
2024-12-02T18:30:33,192 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53502 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741867_1050] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53502 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,193 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:33,195 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33679
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53512 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051 to mirror 127.0.0.1:33679
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,195 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:33,195 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051
2024-12-02T18:30:33,195 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53512 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:33,196 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53512 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53512 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,196 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:33,198 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40249
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,198 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53518 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052 to mirror 127.0.0.1:40249
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,199 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:33,199 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052
2024-12-02T18:30:33,199 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53518 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:33,199 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53518 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53518 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,199 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:33,201 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,201 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:33,201 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741870_1053
2024-12-02T18:30:33,201 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:33,202 WARN  [IPC Server handler 4 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:33,202 WARN  [IPC Server handler 4 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:33,202 WARN  [IPC Server handler 4 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:33,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741871_1054 (size=6027)
2024-12-02T18:30:33,407 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,564 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,564 WARN  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]
2024-12-02T18:30:33,564 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C42043%2C1733164210037:(num 1733164231545) roll requested
2024-12-02T18:30:33,565 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164233565
2024-12-02T18:30:33,569 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42381
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,569 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53528 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055 to mirror 127.0.0.1:42381
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,570 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:33,570 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53528 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-02T18:30:33,570 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055
2024-12-02T18:30:33,570 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53528 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741872_1055] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53528 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,571 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:33,572 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,573 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:33,573 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741873_1056
2024-12-02T18:30:33,577 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:33,578 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,579 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:33,579 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741874_1057
2024-12-02T18:30:33,579 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:33,582 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33679
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:33,582 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53534 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058 to mirror 127.0.0.1:33679
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,583 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:33,583 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058
2024-12-02T18:30:33,583 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53534 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-02T18:30:33,583 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53534 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53534 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,583 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:33,584 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:33,584 WARN  [IPC Server handler 2 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:33,584 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:33,587 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:33,588 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:33,588 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:33,588 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:33,588 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:33,588 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164231545 with entries=13, filesize=11.78 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164233565
2024-12-02T18:30:33,589 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42809:42809)]
2024-12-02T18:30:33,589 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:33,589 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164231545 is not closed yet, will try archiving it next time
2024-12-02T18:30:33,589 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164227526 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C42043%2C1733164210037.1733164227526
2024-12-02T18:30:33,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741856_1039 (size=12066)
2024-12-02T18:30:33,591 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 is not closed yet, will try archiving it next time
2024-12-02T18:30:33,606 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a982b80830094c528567738edbcd1642
2024-12-02T18:30:33,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a982b80830094c528567738edbcd1642 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642
2024-12-02T18:30:33,622 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642, entries=1, sequenceid=45, filesize=5.9 K
2024-12-02T18:30:33,623 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f395c8610517974fa838bf7ff6019549 in 439ms, sequenceid=45, compaction requested=false
2024-12-02T18:30:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-02T18:30:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 because midkey is the same as first or last row
2024-12-02T18:30:33,955 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@63aed152[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741841_1024 to 127.0.0.1:33679 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:33,955 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741861_1044 to 127.0.0.1:40249 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:34,170 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:34,606 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-02T18:30:34,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a2968de9ae144a90bd6de393bccf3388 is 1079, key is tmprow/info:/1733164234605/Put/seqid=0
2024-12-02T18:30:34,615 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:34,616 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:34,616 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741877_1060
2024-12-02T18:30:34,617 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:34,618 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:34,618 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:34,618 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741878_1061
2024-12-02T18:30:34,619 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:34,620 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:34,620 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:34,620 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741879_1062
2024-12-02T18:30:34,621 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:34,622 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:34,622 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:34,622 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741880_1063
2024-12-02T18:30:34,623 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:34,624 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:34,624 WARN  [IPC Server handler 2 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:34,624 WARN  [IPC Server handler 2 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:34,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741881_1064 (size=6027)
2024-12-02T18:30:35,028 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a2968de9ae144a90bd6de393bccf3388
2024-12-02T18:30:35,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/a2968de9ae144a90bd6de393bccf3388 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388
2024-12-02T18:30:35,045 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388, entries=1, sequenceid=55, filesize=5.9 K
2024-12-02T18:30:35,046 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f395c8610517974fa838bf7ff6019549 in 440ms, sequenceid=55, compaction requested=true
2024-12-02T18:30:35,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:35,046 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-12-02T18:30:35,047 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:35,047 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 because midkey is the same as first or last row
2024-12-02T18:30:35,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f395c8610517974fa838bf7ff6019549:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:30:35,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:30:35,047 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:30:35,049 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:30:35,049 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1541): f395c8610517974fa838bf7ff6019549/info is initiating minor compaction (all files)
2024-12-02T18:30:35,049 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f395c8610517974fa838bf7ff6019549/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:35,049 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388] into tmpdir=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp, totalSize=29.3 K
2024-12-02T18:30:35,050 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689597003a474bb59ac6231f967fc0f5, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733164225597
2024-12-02T18:30:35,050 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting a982b80830094c528567738edbcd1642, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733164233183
2024-12-02T18:30:35,050 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2968de9ae144a90bd6de393bccf3388, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733164234605
2024-12-02T18:30:35,067 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f395c8610517974fa838bf7ff6019549#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:30:35,068 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/1ca943a621614f49ba0873fd44014fcd is 1080, key is row0002/info:/1733164225597/Put/seqid=0
2024-12-02T18:30:35,070 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,070 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:35,070 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741882_1065
2024-12-02T18:30:35,070 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:35,072 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,072 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:35,072 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741883_1066
2024-12-02T18:30:35,072 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:35,074 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40249
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,074 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53564 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067 to mirror 127.0.0.1:40249
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:35,074 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]) is bad.
2024-12-02T18:30:35,074 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067
2024-12-02T18:30:35,075 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53564 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:35,075 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53564 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53564 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:35,075 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]
2024-12-02T18:30:35,078 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33679
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,077 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53580 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6]'}, localName='127.0.0.1:45691', datanodeUuid='c0d54796-ec0c-47bb-a1e5-f993bf3f04f1', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068 to mirror 127.0.0.1:33679
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:35,078 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK], DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]) is bad.
2024-12-02T18:30:35,078 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53580 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-02T18:30:35,078 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068
2024-12-02T18:30:35,078 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1173190388_22 at /127.0.0.1:53580 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:45691:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53580 dst: /127.0.0.1:45691
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:35,078 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33679,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]
2024-12-02T18:30:35,079 WARN  [IPC Server handler 4 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-02T18:30:35,079 WARN  [IPC Server handler 4 on default port 40269 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-02T18:30:35,079 WARN  [IPC Server handler 4 on default port 40269 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-02T18:30:35,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741886_1069 (size=18097)
2024-12-02T18:30:35,408 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,496 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/1ca943a621614f49ba0873fd44014fcd as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/1ca943a621614f49ba0873fd44014fcd
2024-12-02T18:30:35,514 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f395c8610517974fa838bf7ff6019549/info of f395c8610517974fa838bf7ff6019549 into 1ca943a621614f49ba0873fd44014fcd(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:30:35,514 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:35,514 INFO  [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549., storeName=f395c8610517974fa838bf7ff6019549/info, priority=13, startTime=1733164235047; duration=0sec
2024-12-02T18:30:35,514 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-02T18:30:35,514 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:35,514 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/1ca943a621614f49ba0873fd44014fcd because midkey is the same as first or last row
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/1ca943a621614f49ba0873fd44014fcd because midkey is the same as first or last row
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/1ca943a621614f49ba0873fd44014fcd because midkey is the same as first or last row
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:30:35,515 DEBUG [RS:0;95451c90bed2:42043-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f395c8610517974fa838bf7ff6019549:info
2024-12-02T18:30:35,590 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:35,590 WARN  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-02T18:30:35,632 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:35,636 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:35,637 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:35,637 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:35,637 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:30:35,638 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e00c8cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:35,638 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22c6c03b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:35,737 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3eb1ba79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/java.io.tmpdir/jetty-localhost-46153-hadoop-hdfs-3_4_1-tests_jar-_-any-1536384877316448436/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:35,738 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@117da513{HTTP/1.1, (http/1.1)}{localhost:46153}
2024-12-02T18:30:35,738 INFO  [Time-limited test {}] server.Server(415): Started @131124ms
2024-12-02T18:30:35,739 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:35,954 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@63aed152[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741871_1054 to 127.0.0.1:42381 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:35,954 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741866_1049 to 127.0.0.1:33679 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:36,170 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:36,208 WARN  [Thread-983 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:36,216 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x145fe6eed528a140 with lease ID 0x5c8e6f81af6c585: from storage DS-e89f1db7-47ae-48ed-961d-140e674870de node DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:30:36,217 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x145fe6eed528a140 with lease ID 0x5c8e6f81af6c585: from storage DS-63fc2016-b1ae-4261-962a-a3ab68f4e3d3 node DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:36,955 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@63aed152[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741856_1039 to 127.0.0.1:40249 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:36,955 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741881_1064 to 127.0.0.1:42381 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:37,408 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:37,590 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:38,171 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:38,954 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45691, datanodeUuid=c0d54796-ec0c-47bb-a1e5-f993bf3f04f1, infoPort=42809, infoSecurePort=0, ipcPort=40735, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741886_1069 to 127.0.0.1:42381 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:39,408 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:39,590 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:39,835 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:30:40,171 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,405 ERROR [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData-prefix:95451c90bed2,43127,1733164209855 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,405 WARN  [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData-prefix:95451c90bed2,43127,1733164209855 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,406 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C43127%2C1733164209855:(num 1733164210188) roll requested
2024-12-02T18:30:40,406 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43127%2C1733164209855.1733164240406
2024-12-02T18:30:40,409 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,410 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]) is bad.
2024-12-02T18:30:40,410 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741887_1070
2024-12-02T18:30:40,413 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42381,DS-70407dbe-8205-4621-ba2e-2e1699c87aac,DISK]
2024-12-02T18:30:40,416 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41211
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,416 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-45597091_22 at /127.0.0.1:57206 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data4]'}, localName='127.0.0.1:37063', datanodeUuid='d1e32ab3-9f16-4281-a5f9-533d4a7d7e90', xmitsInProgress=0}:Exception transferring block BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071 to mirror 127.0.0.1:41211
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:40,416 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37063,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK], DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:40,416 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071
2024-12-02T18:30:40,416 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-45597091_22 at /127.0.0.1:57206 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-02T18:30:40,416 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-45597091_22 at /127.0.0.1:57206 [Receiving block BP-209545290-172.17.0.3-1733164207654:blk_1073741888_1071] {}] datanode.DataXceiver(331): 127.0.0.1:37063:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57206 dst: /127.0.0.1:37063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:40,417 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:40,422 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:40,422 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:40,422 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:40,422 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:40,423 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:40,423 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164240406
2024-12-02T18:30:40,423 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,424 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:40,424 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188
2024-12-02T18:30:40,424 WARN  [IPC Server handler 0 on default port 40269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-12-02T18:30:40,425 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 after 1ms
2024-12-02T18:30:40,430 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42809:42809),(127.0.0.1/127.0.0.1:35995:35995)]
2024-12-02T18:30:40,430 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 is not closed yet, will try archiving it next time
2024-12-02T18:30:41,409 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:41,591 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:43,409 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:43,591 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:44,426 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 after 4002ms
2024-12-02T18:30:45,211 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5e665a6b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741836_1012 to 127.0.0.1:42381 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:45,211 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@16282f68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741832_1008 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:45,410 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:45,592 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:46,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@16282f68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741828_1004 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:46,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:46,230 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@7607d027 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:40249,null,null])
java.net.ConnectException: Call From 95451c90bed2/172.17.0.3 to localhost:32931 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-02T18:30:46,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741833_1019 (size=455)
2024-12-02T18:30:46,542 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164210667 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C42043%2C1733164210037.1733164210667
2024-12-02T18:30:46,544 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164231545 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C42043%2C1733164210037.1733164231545
2024-12-02T18:30:47,410 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:47,592 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:48,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5e665a6b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741825_1001 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:48,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@16282f68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741833_1019 to 127.0.0.1:42381 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:49,385 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.1733164249385
2024-12-02T18:30:49,391 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,392 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,392 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,392 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,392 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,392 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164233565 with entries=14, filesize=12.95 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164249385
2024-12-02T18:30:49,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741876_1059 (size=13268)
2024-12-02T18:30:49,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42809:42809),(127.0.0.1/127.0.0.1:35995:35995)]
2024-12-02T18:30:49,401 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164233565 is not closed yet, will try archiving it next time
2024-12-02T18:30:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42043 {}] regionserver.HRegion(8855): Flush requested on f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:49,409 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-02T18:30:49,410 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/23990582f32d4593a7502158b56c40ef is 1080, key is row0013/info:/1733164249402/Put/seqid=0
2024-12-02T18:30:49,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741891_1075 (size=9267)
2024-12-02T18:30:49,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741891_1075 (size=9267)
2024-12-02T18:30:49,424 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/23990582f32d4593a7502158b56c40ef
2024-12-02T18:30:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/23990582f32d4593a7502158b56c40ef as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/23990582f32d4593a7502158b56c40ef
2024-12-02T18:30:49,439 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/23990582f32d4593a7502158b56c40ef, entries=4, sequenceid=66, filesize=9.0 K
2024-12-02T18:30:49,441 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for f395c8610517974fa838bf7ff6019549 in 32ms, sequenceid=66, compaction requested=false
2024-12-02T18:30:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f395c8610517974fa838bf7ff6019549:

2024-12-02T18:30:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.7 K, sizeToCheck=16.0 K
2024-12-02T18:30:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:30:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/1ca943a621614f49ba0873fd44014fcd because midkey is the same as first or last row
2024-12-02T18:30:49,592 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,592 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-02T18:30:49,631 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:30:49,631 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:49,631 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:49,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:49,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:49,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:30:49,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=221351778, stopped=false
2024-12-02T18:30:49,632 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,43127,1733164209855
2024-12-02T18:30:49,632 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:49,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:49,671 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:49,671 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:30:49,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:49,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:49,672 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,42043,1733164210037' *****
2024-12-02T18:30:49,672 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:30:49,672 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,37951,1733164211301' *****
2024-12-02T18:30:49,672 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:30:49,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:30:49,672 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:30:49,672 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:30:49,672 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:30:49,672 INFO  [RS:0;95451c90bed2:42043 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:30:49,672 INFO  [RS:0;95451c90bed2:42043 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,37951,1733164211301
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:49,672 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(3091): Received CLOSE for f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:49,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:49,672 INFO  [RS:1;95451c90bed2:37951 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;95451c90bed2:37951.
2024-12-02T18:30:49,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:49,673 DEBUG [RS:1;95451c90bed2:37951 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:49,673 DEBUG [RS:1;95451c90bed2:37951 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,42043,1733164210037
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:49,673 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,37951,1733164211301; all regions closed.
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:42043.
2024-12-02T18:30:49,673 DEBUG [RS:0;95451c90bed2:42043 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:30:49,673 DEBUG [RS:0;95451c90bed2:42043 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:30:49,673 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:30:49,673 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,673 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,673 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,673 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,673 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,673 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f395c8610517974fa838bf7ff6019549, disabling compactions & flushes
2024-12-02T18:30:49,674 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:49,674 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:49,674 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549. after waiting 0 ms
2024-12-02T18:30:49,674 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:49,674 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f395c8610517974fa838bf7ff6019549 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-02T18:30:49,675 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-02T18:30:49,675 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f395c8610517974fa838bf7ff6019549=TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.}
2024-12-02T18:30:49,675 DEBUG [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f395c8610517974fa838bf7ff6019549
2024-12-02T18:30:49,675 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:30:49,675 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:30:49,675 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:30:49,675 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:30:49,675 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:30:49,675 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-02T18:30:49,676 ERROR [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df-prefix:95451c90bed2,42043,1733164210037.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,676 WARN  [FSHLog-0-hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df-prefix:95451c90bed2,42043,1733164210037.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,676 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C42043%2C1733164210037.meta:.meta(num 1733164211081) roll requested
2024-12-02T18:30:49,676 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42043%2C1733164210037.meta.1733164249676.meta
2024-12-02T18:30:49,677 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,677 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,677 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
2024-12-02T18:30:49,678 WARN  [IPC Server handler 4 on default port 40269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741837_1013
2024-12-02T18:30:49,678 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 after 1ms
2024-12-02T18:30:49,679 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9b589d14a9d841bfa6947a549ad216b7 is 1080, key is row0016/info:/1733164249410/Put/seqid=0
2024-12-02T18:30:49,680 WARN  [Thread-1031 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1077
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,681 WARN  [Thread-1031 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741892_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:45691,DS-f0870539-7f8d-4124-8e24-f89d356151d6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:49,681 WARN  [Thread-1031 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741892_1077
2024-12-02T18:30:49,681 WARN  [Thread-1031 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:49,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741893_1078 (size=13583)
2024-12-02T18:30:49,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741893_1078 (size=13583)
2024-12-02T18:30:49,687 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9b589d14a9d841bfa6947a549ad216b7
2024-12-02T18:30:49,689 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,689 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,689 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,689 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,689 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,690 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164249676.meta
2024-12-02T18:30:49,690 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,690 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40249,DS-ccfbed4f-cfdd-48ee-b7ed-ed9afca52f60,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,690 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
2024-12-02T18:30:49,691 WARN  [IPC Server handler 2 on default port 40269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741834_1010
2024-12-02T18:30:49,691 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta after 1ms
2024-12-02T18:30:49,697 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35995:35995),(127.0.0.1/127.0.0.1:42809:42809)]
2024-12-02T18:30:49,697 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta is not closed yet, will try archiving it next time
2024-12-02T18:30:49,701 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/.tmp/info/9b589d14a9d841bfa6947a549ad216b7 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9b589d14a9d841bfa6947a549ad216b7
2024-12-02T18:30:49,708 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9b589d14a9d841bfa6947a549ad216b7, entries=8, sequenceid=77, filesize=13.3 K
2024-12-02T18:30:49,709 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for f395c8610517974fa838bf7ff6019549 in 35ms, sequenceid=77, compaction requested=true
2024-12-02T18:30:49,712 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388] to archive
2024-12-02T18:30:49,714 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:30:49,715 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/info/b7e6ba85f2d5492c95b09a22c57b5eca is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549./info:regioninfo/1733164211804/Put/seqid=0
2024-12-02T18:30:49,716 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/3a5a2a0f4ef04bcfbf0695e0c586c580
2024-12-02T18:30:49,716 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:30:49,717 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-209545290-172.17.0.3-1733164207654:blk_1073741895_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK], DatanodeInfoWithStorage[127.0.0.1:37063,DS-e89f1db7-47ae-48ed-961d-140e674870de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]) is bad.
2024-12-02T18:30:49,717 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-209545290-172.17.0.3-1733164207654:blk_1073741895_1081
2024-12-02T18:30:49,717 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/d91374a3a3e94d3ba55947f8c9153166
2024-12-02T18:30:49,717 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41211,DS-24b27e8b-9ef7-42d8-a981-3faf1236491b,DISK]
2024-12-02T18:30:49,719 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/689597003a474bb59ac6231f967fc0f5
2024-12-02T18:30:49,721 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/9f1e7e43c7a84c97a46dc698efc5a4c4
2024-12-02T18:30:49,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741896_1082 (size=7089)
2024-12-02T18:30:49,722 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741896_1082 (size=7089)
2024-12-02T18:30:49,722 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/info/b7e6ba85f2d5492c95b09a22c57b5eca
2024-12-02T18:30:49,723 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a982b80830094c528567738edbcd1642
2024-12-02T18:30:49,725 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/info/a2968de9ae144a90bd6de393bccf3388
2024-12-02T18:30:49,725 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=95451c90bed2:43127 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-02T18:30:49,725 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3a5a2a0f4ef04bcfbf0695e0c586c580=10347, d91374a3a3e94d3ba55947f8c9153166=12506, 689597003a474bb59ac6231f967fc0f5=17994, 9f1e7e43c7a84c97a46dc698efc5a4c4=6027, a982b80830094c528567738edbcd1642=6027, a2968de9ae144a90bd6de393bccf3388=6027]
2024-12-02T18:30:49,730 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f395c8610517974fa838bf7ff6019549/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1
2024-12-02T18:30:49,730 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:49,731 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f395c8610517974fa838bf7ff6019549:
Waiting for close lock at 1733164249673Running coprocessor pre-close hooks at 1733164249673Disabling compacts and flushes for region at 1733164249673Disabling writes for close at 1733164249674 (+1 ms)Obtaining lock to block concurrent updates at 1733164249674Preparing flush snapshotting stores in f395c8610517974fa838bf7ff6019549 at 1733164249674Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549., syncing WAL and waiting on mvcc, flushsize=dataSize=8607, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1733164249674Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549. at 1733164249675 (+1 ms)Flushing f395c8610517974fa838bf7ff6019549/info: creating writer at 1733164249675Flushing f395c8610517974fa838bf7ff6019549/info: appending metadata at 1733164249679 (+4 ms)Flushing f395c8610517974fa838bf7ff6019549/info: closing flushed file at 1733164249679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c5efaa0: reopening flushed file at 1733164249700 (+21 ms)Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for f395c8610517974fa838bf7ff6019549 in 35ms, sequenceid=77, compaction requested=true at 1733164249709 (+9 ms)Writing region close event to WAL at 1733164249726 (+17 ms)Running coprocessor post-close hooks at 1733164249730 (+4 ms)Closed at 1733164249730
2024-12-02T18:30:49,731 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733164211436.f395c8610517974fa838bf7ff6019549.
2024-12-02T18:30:49,745 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/ns/d7cdadce47c840ecacc2c8f341e236e5 is 43, key is default/ns:d/1733164211191/Put/seqid=0
2024-12-02T18:30:49,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741897_1083 (size=5153)
2024-12-02T18:30:49,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741897_1083 (size=5153)
2024-12-02T18:30:49,751 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/ns/d7cdadce47c840ecacc2c8f341e236e5
2024-12-02T18:30:49,771 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/table/16ceac7d9ec645c692775dc341775eb3 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733164211817/Put/seqid=0
2024-12-02T18:30:49,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741898_1084 (size=5424)
2024-12-02T18:30:49,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741898_1084 (size=5424)
2024-12-02T18:30:49,777 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/table/16ceac7d9ec645c692775dc341775eb3
2024-12-02T18:30:49,783 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/info/b7e6ba85f2d5492c95b09a22c57b5eca as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/info/b7e6ba85f2d5492c95b09a22c57b5eca
2024-12-02T18:30:49,790 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/info/b7e6ba85f2d5492c95b09a22c57b5eca, entries=10, sequenceid=11, filesize=6.9 K
2024-12-02T18:30:49,791 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/ns/d7cdadce47c840ecacc2c8f341e236e5 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/ns/d7cdadce47c840ecacc2c8f341e236e5
2024-12-02T18:30:49,794 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.1733164233565 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C42043%2C1733164210037.1733164233565
2024-12-02T18:30:49,799 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/ns/d7cdadce47c840ecacc2c8f341e236e5, entries=2, sequenceid=11, filesize=5.0 K
2024-12-02T18:30:49,800 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/.tmp/table/16ceac7d9ec645c692775dc341775eb3 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/table/16ceac7d9ec645c692775dc341775eb3
2024-12-02T18:30:49,807 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/table/16ceac7d9ec645c692775dc341775eb3, entries=2, sequenceid=11, filesize=5.3 K
2024-12-02T18:30:49,809 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 133ms, sequenceid=11, compaction requested=false
2024-12-02T18:30:49,814 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-02T18:30:49,814 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:30:49,814 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:49,815 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164249675Running coprocessor pre-close hooks at 1733164249675Disabling compacts and flushes for region at 1733164249675Disabling writes for close at 1733164249675Obtaining lock to block concurrent updates at 1733164249675Preparing flush snapshotting stores in 1588230740 at 1733164249675Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733164249676 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733164249698 (+22 ms)Flushing 1588230740/info: creating writer at 1733164249698Flushing 1588230740/info: appending metadata at 1733164249714 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733164249714Flushing 1588230740/ns: creating writer at 1733164249729 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733164249745 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733164249745Flushing 1588230740/table: creating writer at 1733164249757 (+12 ms)Flushing 1588230740/table: appending metadata at 1733164249770 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733164249770Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c571a: reopening flushed file at 1733164249782 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@332d7ce: reopening flushed file at 1733164249790 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@323d477b: reopening flushed file at 1733164249799 (+9 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 133ms, sequenceid=11, compaction requested=false at 1733164249809 (+10 ms)Writing region close event to WAL at 1733164249810 (+1 ms)Running coprocessor post-close hooks at 1733164249814 (+4 ms)Closed at 1733164249814
2024-12-02T18:30:49,815 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:30:49,875 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,42043,1733164210037; all regions closed.
2024-12-02T18:30:49,876 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,876 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,876 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,876 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,876 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:49,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741894_1079 (size=825)
2024-12-02T18:30:49,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741894_1079 (size=825)
2024-12-02T18:30:50,405 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-02T18:30:50,406 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-02T18:30:50,529 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:50,546 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-02T18:30:50,546 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-02T18:30:50,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741876_1059 (size=13268)
2024-12-02T18:30:51,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5e665a6b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741831_1007 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:51,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741835_1011 (size=393)
2024-12-02T18:30:51,407 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:51,562 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-02T18:30:51,562 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-02T18:30:51,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-02T18:30:51,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:30:51,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:30:52,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5e665a6b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741827_1003 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:52,212 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@16282f68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37063, datanodeUuid=d1e32ab3-9f16-4281-a5f9-533d4a7d7e90, infoPort=35995, infoSecurePort=0, ipcPort=43373, storageInfo=lv=-57;cid=testClusterID;nsid=447832675;c=1733164207654):Failed to transfer BP-209545290-172.17.0.3-1733164207654:blk_1073741829_1005 to 127.0.0.1:41211 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:53,679 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 after 4002ms
2024-12-02T18:30:53,692 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta after 4002ms
2024-12-02T18:30:54,677 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-02T18:30:54,679 DEBUG [RS:1;95451c90bed2:37951 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs
2024-12-02T18:30:54,679 INFO  [RS:1;95451c90bed2:37951 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C37951%2C1733164211301:(num 1733164211537)
2024-12-02T18:30:54,679 DEBUG [RS:1;95451c90bed2:37951 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:54,679 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:30:54,680 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:54,680 INFO  [RS:1;95451c90bed2:37951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37951
2024-12-02T18:30:54,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:54,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,37951,1733164211301
2024-12-02T18:30:54,740 INFO  [RS:1;95451c90bed2:37951 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:54,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:54,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,755 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,37951,1733164211301]
2024-12-02T18:30:54,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:54,765 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,37951,1733164211301 already deleted, retry=false
2024-12-02T18:30:54,765 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,37951,1733164211301 expired; onlineServers=1
2024-12-02T18:30:54,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:54,855 INFO  [RS:1;95451c90bed2:37951 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:54,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37951-0x101974f125a0002, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:54,855 INFO  [RS:1;95451c90bed2:37951 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,37951,1733164211301; zookeeper connection closed.
2024-12-02T18:30:54,855 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@62be20cb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@62be20cb
2024-12-02T18:30:54,876 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-02T18:30:54,880 DEBUG [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs
2024-12-02T18:30:54,880 INFO  [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C42043%2C1733164210037.meta:.meta(num 1733164249676)
2024-12-02T18:30:54,881 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:54,881 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:54,881 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:54,881 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:54,881 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:54,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741890_1074 (size=14682)
2024-12-02T18:30:54,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741890_1074 (size=14682)
2024-12-02T18:30:54,896 DEBUG [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs
2024-12-02T18:30:54,896 INFO  [RS:0;95451c90bed2:42043 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C42043%2C1733164210037:(num 1733164249385)
2024-12-02T18:30:54,896 DEBUG [RS:0;95451c90bed2:42043 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:30:54,896 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:30:54,896 INFO  [RS:0;95451c90bed2:42043 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:54,896 INFO  [RS:0;95451c90bed2:42043 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:54,897 INFO  [RS:0;95451c90bed2:42043 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:54,897 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:54,897 INFO  [RS:0;95451c90bed2:42043 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42043
2024-12-02T18:30:54,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:30:54,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,42043,1733164210037
2024-12-02T18:30:54,908 INFO  [RS:0;95451c90bed2:42043 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:54,919 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,42043,1733164210037]
2024-12-02T18:30:54,929 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,42043,1733164210037 already deleted, retry=false
2024-12-02T18:30:54,929 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,42043,1733164210037 expired; onlineServers=0
2024-12-02T18:30:54,929 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,43127,1733164209855' *****
2024-12-02T18:30:54,929 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:30:54,929 INFO  [M:0;95451c90bed2:43127 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:30:54,930 INFO  [M:0;95451c90bed2:43127 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:30:54,930 DEBUG [M:0;95451c90bed2:43127 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:30:54,930 DEBUG [M:0;95451c90bed2:43127 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:30:54,930 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:30:54,930 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164210406 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164210406,5,FailOnTimeoutGroup]
2024-12-02T18:30:54,930 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164210406 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164210406,5,FailOnTimeoutGroup]
2024-12-02T18:30:54,930 INFO  [M:0;95451c90bed2:43127 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:30:54,930 INFO  [M:0;95451c90bed2:43127 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:30:54,930 DEBUG [M:0;95451c90bed2:43127 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:30:54,930 INFO  [M:0;95451c90bed2:43127 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:30:54,931 INFO  [M:0;95451c90bed2:43127 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:30:54,931 INFO  [M:0;95451c90bed2:43127 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:30:54,931 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:30:54,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:54,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:54,940 DEBUG [M:0;95451c90bed2:43127 {}] zookeeper.ZKUtil(347): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:30:54,940 WARN  [M:0;95451c90bed2:43127 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:30:54,941 INFO  [M:0;95451c90bed2:43127 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/.lastflushedseqids
2024-12-02T18:30:54,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741899_1085 (size=130)
2024-12-02T18:30:54,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741899_1085 (size=130)
2024-12-02T18:30:54,948 INFO  [M:0;95451c90bed2:43127 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:30:54,948 INFO  [M:0;95451c90bed2:43127 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:30:54,948 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:54,948 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:54,948 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:54,948 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:54,948 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:54,948 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-12-02T18:30:54,967 DEBUG [M:0;95451c90bed2:43127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8f7ac15056d64c86b42dc7b568f46915 is 82, key is hbase:meta,,1/info:regioninfo/1733164211143/Put/seqid=0
2024-12-02T18:30:54,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741900_1086 (size=5672)
2024-12-02T18:30:54,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741900_1086 (size=5672)
2024-12-02T18:30:54,991 INFO  [M:0;95451c90bed2:43127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8f7ac15056d64c86b42dc7b568f46915
2024-12-02T18:30:55,011 DEBUG [M:0;95451c90bed2:43127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00563bc15d2c4a2daafc7eab80b45f53 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733164211825/Put/seqid=0
2024-12-02T18:30:55,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741901_1087 (size=6255)
2024-12-02T18:30:55,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741901_1087 (size=6255)
2024-12-02T18:30:55,017 INFO  [M:0;95451c90bed2:43127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00563bc15d2c4a2daafc7eab80b45f53
2024-12-02T18:30:55,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:55,019 INFO  [RS:0;95451c90bed2:42043 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:55,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42043-0x101974f125a0001, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:55,019 INFO  [RS:0;95451c90bed2:42043 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,42043,1733164210037; zookeeper connection closed.
2024-12-02T18:30:55,019 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36f299fc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36f299fc
2024-12-02T18:30:55,019 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-02T18:30:55,022 INFO  [M:0;95451c90bed2:43127 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00563bc15d2c4a2daafc7eab80b45f53
2024-12-02T18:30:55,038 DEBUG [M:0;95451c90bed2:43127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2fe82c625b41c3ba320fd5d0134693 is 69, key is 95451c90bed2,37951,1733164211301/rs:state/1733164211381/Put/seqid=0
2024-12-02T18:30:55,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741902_1088 (size=5224)
2024-12-02T18:30:55,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741902_1088 (size=5224)
2024-12-02T18:30:55,043 INFO  [M:0;95451c90bed2:43127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2fe82c625b41c3ba320fd5d0134693
2024-12-02T18:30:55,062 DEBUG [M:0;95451c90bed2:43127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/606fca95e7724fa0a5eb365926a0cd16 is 52, key is load_balancer_on/state:d/1733164211284/Put/seqid=0
2024-12-02T18:30:55,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741903_1089 (size=5056)
2024-12-02T18:30:55,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741903_1089 (size=5056)
2024-12-02T18:30:55,067 INFO  [M:0;95451c90bed2:43127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/606fca95e7724fa0a5eb365926a0cd16
2024-12-02T18:30:55,075 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8f7ac15056d64c86b42dc7b568f46915 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8f7ac15056d64c86b42dc7b568f46915
2024-12-02T18:30:55,081 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8f7ac15056d64c86b42dc7b568f46915, entries=8, sequenceid=60, filesize=5.5 K
2024-12-02T18:30:55,083 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00563bc15d2c4a2daafc7eab80b45f53 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00563bc15d2c4a2daafc7eab80b45f53
2024-12-02T18:30:55,089 INFO  [M:0;95451c90bed2:43127 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00563bc15d2c4a2daafc7eab80b45f53
2024-12-02T18:30:55,089 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00563bc15d2c4a2daafc7eab80b45f53, entries=6, sequenceid=60, filesize=6.1 K
2024-12-02T18:30:55,090 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8b2fe82c625b41c3ba320fd5d0134693 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2fe82c625b41c3ba320fd5d0134693
2024-12-02T18:30:55,096 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8b2fe82c625b41c3ba320fd5d0134693, entries=2, sequenceid=60, filesize=5.1 K
2024-12-02T18:30:55,098 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/606fca95e7724fa0a5eb365926a0cd16 as hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/606fca95e7724fa0a5eb365926a0cd16
2024-12-02T18:30:55,103 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/606fca95e7724fa0a5eb365926a0cd16, entries=1, sequenceid=60, filesize=4.9 K
2024-12-02T18:30:55,104 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=60, compaction requested=false
2024-12-02T18:30:55,106 INFO  [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:55,106 DEBUG [M:0;95451c90bed2:43127 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164254948Disabling compacts and flushes for region at 1733164254948Disabling writes for close at 1733164254948Obtaining lock to block concurrent updates at 1733164254948Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164254948Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733164254949 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164254950 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164254950Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164254967 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164254967Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164254996 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164255011 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164255011Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164255023 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164255037 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164255037Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164255048 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164255062 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164255062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cfda31b: reopening flushed file at 1733164255073 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b0eba55: reopening flushed file at 1733164255082 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52cff06c: reopening flushed file at 1733164255089 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b3d19ca: reopening flushed file at 1733164255097 (+8 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=60, compaction requested=false at 1733164255104 (+7 ms)Writing region close event to WAL at 1733164255106 (+2 ms)Closed at 1733164255106
2024-12-02T18:30:55,106 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:55,106 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:55,106 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:55,106 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:55,106 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:30:55,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37063 is added to blk_1073741889_1072 (size=1045)
2024-12-02T18:30:55,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45691 is added to blk_1073741889_1072 (size=1045)
2024-12-02T18:30:55,264 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:30:55,284 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,292 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:30:55,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:55,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:56,235 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2630920f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:40249,null,null])
java.net.ConnectException: Call From 95451c90bed2/172.17.0.3 to localhost:32931 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-02T18:30:56,435 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/WALs/95451c90bed2,43127,1733164209855/95451c90bed2%2C43127%2C1733164209855.1733164210188 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/oldWALs/95451c90bed2%2C43127%2C1733164209855.1733164210188
2024-12-02T18:30:56,439 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/MasterData/oldWALs/95451c90bed2%2C43127%2C1733164209855.1733164210188 to hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/oldWALs/95451c90bed2%2C43127%2C1733164209855.1733164210188$masterlocalwal$
2024-12-02T18:30:56,439 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:30:56,439 INFO  [M:0;95451c90bed2:43127 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:30:56,439 INFO  [M:0;95451c90bed2:43127 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43127
2024-12-02T18:30:56,440 INFO  [M:0;95451c90bed2:43127 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:30:56,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:56,613 INFO  [M:0;95451c90bed2:43127 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:30:56,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43127-0x101974f125a0000, quorum=127.0.0.1:56946, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:30:56,616 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3eb1ba79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:56,616 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@117da513{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:56,617 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:56,617 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22c6c03b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:56,617 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e00c8cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:56,619 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:56,619 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:56,619 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid d1e32ab3-9f16-4281-a5f9-533d4a7d7e90) service to localhost/127.0.0.1:40269
2024-12-02T18:30:56,619 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:56,619 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:40249,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:32931 , LocalHost:localPort 95451c90bed2/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-02T18:30:56,619 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:37063,null,null])
java.io.IOException: No block pool offer service for bpid=BP-209545290-172.17.0.3-1733164207654
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:56,620 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:40249,null,null], DatanodeInfoWithStorage[127.0.0.1:37063,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-209545290-172.17.0.3-1733164207654:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:40249,null,null], DatanodeInfoWithStorage[127.0.0.1:37063,null,null]]
2024-12-02T18:30:56,620 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data3/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:56,620 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37063,null,null])
java.io.IOException: No block pool offer service for bpid=BP-209545290-172.17.0.3-1733164207654
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:56,620 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40249,null,null])
java.io.IOException: No block pool offer service for bpid=BP-209545290-172.17.0.3-1733164207654
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:30:56,620 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@342a6748 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37063,null,null], DatanodeInfoWithStorage[127.0.0.1:40249,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-209545290-172.17.0.3-1733164207654:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37063,null,null], DatanodeInfoWithStorage[127.0.0.1:40249,null,null]]
2024-12-02T18:30:56,620 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data4/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:56,621 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:56,623 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2f0a2519{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:56,624 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68004957{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:56,624 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:56,624 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@719bbb9b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:56,624 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fb911ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:56,625 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:30:56,625 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:30:56,626 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:30:56,626 WARN  [BP-209545290-172.17.0.3-1733164207654 heartbeating to localhost/127.0.0.1:40269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-209545290-172.17.0.3-1733164207654 (Datanode Uuid c0d54796-ec0c-47bb-a1e5-f993bf3f04f1) service to localhost/127.0.0.1:40269
2024-12-02T18:30:56,626 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data5/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:56,626 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/cluster_ed83f767-9aae-522d-d231-52ad0950a4f1/data/data6/current/BP-209545290-172.17.0.3-1733164207654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:30:56,627 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:30:56,632 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c461833{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:56,633 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33e53d1d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:30:56,633 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:30:56,633 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ab5393f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:30:56,633 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@692b8c40{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir/,STOPPED}
2024-12-02T18:30:56,641 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:30:56,671 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:30:56,680 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40269
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40269 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:45303
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40269
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40269
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40269 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45303
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40269
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40269 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40269
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6834bf6848.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40269
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6834bf6848.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40269
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40269 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=448 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 179) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4317 (was 5196)
2024-12-02T18:30:56,687 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=4317
2024-12-02T18:30:56,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.log.dir so I do NOT create it in target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2aba3fe9-0fb9-749a-0313-509faefef9bb/hadoop.tmp.dir so I do NOT create it in target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf, deleteOnExit=true
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/test.cache.data in system properties and HBase conf
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:30:56,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:30:56,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:30:56,688 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:56,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:30:56,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:30:56,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:30:56,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:30:56,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:30:56,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:30:56,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:56,702 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:57,002 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:57,007 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:57,008 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:57,008 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:57,008 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:57,009 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:57,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40c321ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:57,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37ba1ac4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:57,111 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@650740c5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-40541-hadoop-hdfs-3_4_1-tests_jar-_-any-13499454355115173295/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:30:57,111 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a99c341{HTTP/1.1, (http/1.1)}{localhost:40541}
2024-12-02T18:30:57,111 INFO  [Time-limited test {}] server.Server(415): Started @152497ms
2024-12-02T18:30:57,124 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:30:57,399 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:57,402 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:57,404 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:57,404 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:57,404 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:30:57,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@623a52f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:57,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34c1099f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:57,511 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1935e32{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-38023-hadoop-hdfs-3_4_1-tests_jar-_-any-8235957292713894465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:57,511 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@31e2f2e1{HTTP/1.1, (http/1.1)}{localhost:38023}
2024-12-02T18:30:57,512 INFO  [Time-limited test {}] server.Server(415): Started @152898ms
2024-12-02T18:30:57,513 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:57,541 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:30:57,544 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:30:57,545 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:30:57,545 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:30:57,545 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:30:57,546 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25cefabc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:30:57,546 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1118a265{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:30:57,650 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3db1f6e1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-43033-hadoop-hdfs-3_4_1-tests_jar-_-any-11822311557349804839/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:30:57,650 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1be3b2f7{HTTP/1.1, (http/1.1)}{localhost:43033}
2024-12-02T18:30:57,650 INFO  [Time-limited test {}] server.Server(415): Started @153036ms
2024-12-02T18:30:57,652 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:30:57,689 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:57,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:58,689 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:58,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:59,171 WARN  [Thread-1188 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data1/current/BP-2118072695-172.17.0.3-1733164256706/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:59,171 WARN  [Thread-1189 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data2/current/BP-2118072695-172.17.0.3-1733164256706/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:59,190 WARN  [Thread-1152 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:59,192 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ce004f4e74cf9d9 with lease ID 0xae4f8124982e2cb8: Processing first storage report for DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4 from datanode DatanodeRegistration(127.0.0.1:46619, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=41103, infoSecurePort=0, ipcPort=36115, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706)
2024-12-02T18:30:59,192 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ce004f4e74cf9d9 with lease ID 0xae4f8124982e2cb8: from storage DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4 node DatanodeRegistration(127.0.0.1:46619, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=41103, infoSecurePort=0, ipcPort=36115, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:59,193 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ce004f4e74cf9d9 with lease ID 0xae4f8124982e2cb8: Processing first storage report for DS-0c3436e5-7660-4d63-8a7c-d93b2cdc504c from datanode DatanodeRegistration(127.0.0.1:46619, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=41103, infoSecurePort=0, ipcPort=36115, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706)
2024-12-02T18:30:59,193 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ce004f4e74cf9d9 with lease ID 0xae4f8124982e2cb8: from storage DS-0c3436e5-7660-4d63-8a7c-d93b2cdc504c node DatanodeRegistration(127.0.0.1:46619, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=41103, infoSecurePort=0, ipcPort=36115, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:59,299 WARN  [Thread-1199 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data3/current/BP-2118072695-172.17.0.3-1733164256706/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:59,299 WARN  [Thread-1200 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data4/current/BP-2118072695-172.17.0.3-1733164256706/current, will proceed with Du for space computation calculation, 
2024-12-02T18:30:59,319 WARN  [Thread-1175 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:30:59,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59ffa35a7b681b0f with lease ID 0xae4f8124982e2cb9: Processing first storage report for DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c from datanode DatanodeRegistration(127.0.0.1:41125, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=36909, infoSecurePort=0, ipcPort=43335, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706)
2024-12-02T18:30:59,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59ffa35a7b681b0f with lease ID 0xae4f8124982e2cb9: from storage DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c node DatanodeRegistration(127.0.0.1:41125, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=36909, infoSecurePort=0, ipcPort=43335, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:59,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59ffa35a7b681b0f with lease ID 0xae4f8124982e2cb9: Processing first storage report for DS-85008a57-6736-4798-9203-383b979f7cc1 from datanode DatanodeRegistration(127.0.0.1:41125, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=36909, infoSecurePort=0, ipcPort=43335, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706)
2024-12-02T18:30:59,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59ffa35a7b681b0f with lease ID 0xae4f8124982e2cb9: from storage DS-85008a57-6736-4798-9203-383b979f7cc1 node DatanodeRegistration(127.0.0.1:41125, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=36909, infoSecurePort=0, ipcPort=43335, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:30:59,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c
2024-12-02T18:30:59,390 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/zookeeper_0, clientPort=64860, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:30:59,391 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64860
2024-12-02T18:30:59,391 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,393 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:59,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:30:59,403 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5 with version=8
2024-12-02T18:30:59,403 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:30:59,406 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:30:59,407 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:59,408 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39959
2024-12-02T18:30:59,410 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39959 connecting to ZooKeeper ensemble=127.0.0.1:64860
2024-12-02T18:30:59,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399590x0, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:59,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39959-0x101974fd3e80000 connected
2024-12-02T18:30:59,537 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,539 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:59,541 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5, hbase.cluster.distributed=false
2024-12-02T18:30:59,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:59,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39959
2024-12-02T18:30:59,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39959
2024-12-02T18:30:59,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39959
2024-12-02T18:30:59,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39959
2024-12-02T18:30:59,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39959
2024-12-02T18:30:59,560 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:30:59,560 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:30:59,561 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:30:59,562 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44087
2024-12-02T18:30:59,563 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44087 connecting to ZooKeeper ensemble=127.0.0.1:64860
2024-12-02T18:30:59,564 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,566 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440870x0, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:30:59,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:440870x0, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:30:59,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44087-0x101974fd3e80001 connected
2024-12-02T18:30:59,580 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:30:59,580 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:30:59,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:30:59,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:30:59,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44087
2024-12-02T18:30:59,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44087
2024-12-02T18:30:59,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44087
2024-12-02T18:30:59,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44087
2024-12-02T18:30:59,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44087
2024-12-02T18:30:59,596 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:39959
2024-12-02T18:30:59,596 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:59,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:59,600 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:30:59,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,611 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:30:59,611 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,39959,1733164259406 from backup master directory
2024-12-02T18:30:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:59,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:30:59,621 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:30:59,621 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,627 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/hbase.id] with ID: adee941a-93f8-46c1-847f-e73842b3b635
2024-12-02T18:30:59,627 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/.tmp/hbase.id
2024-12-02T18:30:59,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:59,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:30:59,634 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/.tmp/hbase.id]:[hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/hbase.id]
2024-12-02T18:30:59,648 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:30:59,648 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:30:59,650 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-02T18:30:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:59,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:30:59,674 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:30:59,675 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:30:59,676 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:30:59,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:59,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:30:59,684 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store
2024-12-02T18:30:59,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:59,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:59,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:30:59,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:59,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:30:59,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164259692Disabling compacts and flushes for region at 1733164259692Disabling writes for close at 1733164259692Writing region close event to WAL at 1733164259692Closed at 1733164259692
2024-12-02T18:30:59,693 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/.initializing
2024-12-02T18:30:59,693 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:30:59,697 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C39959%2C1733164259406, suffix=, logDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406, archiveDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/oldWALs, maxLogs=10
2024-12-02T18:30:59,698 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C39959%2C1733164259406.1733164259697
2024-12-02T18:30:59,703 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697
2024-12-02T18:30:59,704 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36909:36909),(127.0.0.1/127.0.0.1:41103:41103)]
2024-12-02T18:30:59,714 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:30:59,715 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:30:59,715 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,715 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,716 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:59,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:30:59,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:59,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:30:59,718 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:59,719 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:30:59,719 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:59,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:59,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:59,721 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:30:59,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:59,721 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:59,722 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:30:59,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:30:59,723 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:59,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:30:59,724 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,725 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,725 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,726 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,727 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,727 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:30:59,728 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:30:59,731 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:30:59,731 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=762730, jitterRate=-0.030138954520225525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:30:59,732 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164259715Initializing all the Stores at 1733164259716 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164259716Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164259716Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164259716Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164259716Cleaning up temporary data from old regions at 1733164259727 (+11 ms)Region opened successfully at 1733164259732 (+5 ms)
2024-12-02T18:30:59,732 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:30:59,735 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570773ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:30:59,736 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:30:59,736 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:30:59,736 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:30:59,737 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:30:59,737 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:30:59,738 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:30:59,738 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:30:59,740 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:30:59,741 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:30:59,747 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:30:59,748 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:30:59,749 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:30:59,758 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:30:59,758 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:30:59,760 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:30:59,768 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:30:59,769 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:30:59,779 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:30:59,782 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:30:59,790 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:30:59,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:59,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:30:59,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,800 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,39959,1733164259406, sessionid=0x101974fd3e80000, setting cluster-up flag (Was=false)
2024-12-02T18:30:59,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,853 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:30:59,854 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:30:59,905 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:30:59,907 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,39959,1733164259406
2024-12-02T18:30:59,908 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:30:59,910 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:59,910 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:30:59,910 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:30:59,911 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,39959,1733164259406
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:30:59,912 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164289913
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:30:59,913 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:59,914 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:30:59,914 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:30:59,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:30:59,915 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164259914,5,FailOnTimeoutGroup]
2024-12-02T18:30:59,915 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164259915,5,FailOnTimeoutGroup]
2024-12-02T18:30:59,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:59,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:30:59,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:59,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:30:59,915 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:30:59,915 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:30:59,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:59,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:30:59,922 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:30:59,923 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5
2024-12-02T18:30:59,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:59,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:30:59,985 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(746): ClusterId : adee941a-93f8-46c1-847f-e73842b3b635
2024-12-02T18:30:59,985 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:30:59,990 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:30:59,990 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:31:00,001 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:31:00,001 DEBUG [RS:0;95451c90bed2:44087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682826f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:31:00,014 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:44087
2024-12-02T18:31:00,014 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:31:00,014 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:31:00,014 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:31:00,015 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,39959,1733164259406 with port=44087, startcode=1733164259560
2024-12-02T18:31:00,015 DEBUG [RS:0;95451c90bed2:44087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:31:00,017 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55009, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:31:00,018 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,018 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39959 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,019 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5
2024-12-02T18:31:00,020 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42429
2024-12-02T18:31:00,020 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:31:00,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:31:00,032 DEBUG [RS:0;95451c90bed2:44087 {}] zookeeper.ZKUtil(111): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,032 WARN  [RS:0;95451c90bed2:44087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:31:00,032 INFO  [RS:0;95451c90bed2:44087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:31:00,032 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,033 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,44087,1733164259560]
2024-12-02T18:31:00,035 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:31:00,037 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:31:00,037 INFO  [RS:0;95451c90bed2:44087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:31:00,037 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,037 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:31:00,038 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:31:00,038 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,038 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,038 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:31:00,039 DEBUG [RS:0;95451c90bed2:44087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,040 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,44087,1733164259560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:31:00,062 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:31:00,062 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,44087,1733164259560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,062 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,062 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.Replication(171): 95451c90bed2,44087,1733164259560 started
2024-12-02T18:31:00,082 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,082 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,44087,1733164259560, RpcServer on 95451c90bed2/172.17.0.3:44087, sessionid=0x101974fd3e80001
2024-12-02T18:31:00,082 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:31:00,082 DEBUG [RS:0;95451c90bed2:44087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,082 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,44087,1733164259560'
2024-12-02T18:31:00,082 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,44087,1733164259560'
2024-12-02T18:31:00,083 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:31:00,084 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:31:00,084 DEBUG [RS:0;95451c90bed2:44087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:31:00,084 INFO  [RS:0;95451c90bed2:44087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:31:00,084 INFO  [RS:0;95451c90bed2:44087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:31:00,186 INFO  [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C44087%2C1733164259560, suffix=, logDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560, archiveDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs, maxLogs=32
2024-12-02T18:31:00,187 INFO  [RS:0;95451c90bed2:44087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:00,199 INFO  [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:00,200 DEBUG [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41103:41103),(127.0.0.1/127.0.0.1:36909:36909)]
2024-12-02T18:31:00,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:00,338 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:31:00,339 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:31:00,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:31:00,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:31:00,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:31:00,343 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:31:00,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:31:00,345 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:31:00,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,346 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:31:00,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740
2024-12-02T18:31:00,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740
2024-12-02T18:31:00,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:31:00,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:31:00,349 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:31:00,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:31:00,353 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:31:00,354 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787112, jitterRate=8.654296398162842E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:31:00,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164260337Initializing all the Stores at 1733164260337Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260338 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260338Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164260338Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260338Cleaning up temporary data from old regions at 1733164260349 (+11 ms)Region opened successfully at 1733164260355 (+6 ms)
2024-12-02T18:31:00,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:31:00,355 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:31:00,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:31:00,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:31:00,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:31:00,355 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:31:00,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164260355Disabling compacts and flushes for region at 1733164260355Disabling writes for close at 1733164260355Writing region close event to WAL at 1733164260355Closed at 1733164260355
2024-12-02T18:31:00,357 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:00,357 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:31:00,357 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:31:00,359 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:31:00,360 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:31:00,510 DEBUG [95451c90bed2:39959 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:31:00,511 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,513 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,44087,1733164259560, state=OPENING
2024-12-02T18:31:00,558 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:31:00,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:00,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:00,570 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:00,570 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:31:00,570 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,44087,1733164259560}]
2024-12-02T18:31:00,570 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:00,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:00,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:00,724 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:31:00,727 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43389, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:31:00,731 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:31:00,732 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:31:00,734 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C44087%2C1733164259560.meta, suffix=.meta, logDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560, archiveDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs, maxLogs=32
2024-12-02T18:31:00,735 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta
2024-12-02T18:31:00,745 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta
2024-12-02T18:31:00,746 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41103:41103),(127.0.0.1/127.0.0.1:36909:36909)]
2024-12-02T18:31:00,746 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:31:00,747 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:31:00,747 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:31:00,748 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:31:00,749 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:31:00,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,750 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,750 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:31:00,751 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:31:00,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:31:00,752 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:31:00,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,753 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,753 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:31:00,753 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:31:00,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:00,754 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:31:00,755 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740
2024-12-02T18:31:00,756 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740
2024-12-02T18:31:00,757 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:31:00,758 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:31:00,758 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:31:00,760 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:31:00,761 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792083, jitterRate=0.007185652852058411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:31:00,761 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:31:00,762 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164260747Writing region info on filesystem at 1733164260747Initializing all the Stores at 1733164260748 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260748Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260748Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164260748Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164260748Cleaning up temporary data from old regions at 1733164260758 (+10 ms)Running coprocessor post-open hooks at 1733164260761 (+3 ms)Region opened successfully at 1733164260762 (+1 ms)
2024-12-02T18:31:00,763 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164260724
2024-12-02T18:31:00,765 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:31:00,765 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:31:00,766 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,767 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,44087,1733164259560, state=OPEN
2024-12-02T18:31:00,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:31:00,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:31:00,809 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,44087,1733164259560
2024-12-02T18:31:00,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:00,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:00,814 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:31:00,814 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,44087,1733164259560 in 239 msec
2024-12-02T18:31:00,818 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:31:00,818 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 457 msec
2024-12-02T18:31:00,819 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:00,819 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:31:00,820 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:31:00,820 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,44087,1733164259560, seqNum=-1]
2024-12-02T18:31:00,820 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:31:00,822 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38227, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:31:00,828 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 918 msec
2024-12-02T18:31:00,829 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164260828, completionTime=-1
2024-12-02T18:31:00,829 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:31:00,829 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:31:00,831 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:31:00,831 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164320831
2024-12-02T18:31:00,831 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164380831
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:39959, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,832 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,833 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,834 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.215sec
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:31:00,837 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:31:00,839 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:31:00,839 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:31:00,840 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39959,1733164259406-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:00,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480f741f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:00,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,39959,-1 for getting cluster id
2024-12-02T18:31:00,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:31:00,887 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'adee941a-93f8-46c1-847f-e73842b3b635'
2024-12-02T18:31:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:31:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "adee941a-93f8-46c1-847f-e73842b3b635"

2024-12-02T18:31:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7058ad72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,39959,-1]
2024-12-02T18:31:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:31:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:00,890 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:31:00,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ebcf00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:00,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:31:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,44087,1733164259560, seqNum=-1]
2024-12-02T18:31:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:31:00,894 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:31:00,896 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,39959,1733164259406
2024-12-02T18:31:00,897 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:00,900 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:31:00,900 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-02T18:31:00,900 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-02T18:31:00,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-02T18:31:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 95451c90bed2,39959,1733164259406
2024-12-02T18:31:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@326b4795
2024-12-02T18:31:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-02T18:31:00,904 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-02T18:31:00,904 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-02T18:31:00,904 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-02T18:31:00,905 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:31:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-02T18:31:00,908 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-02T18:31:00,908 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:00,908 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-02T18:31:00,909 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-02T18:31:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:31:00,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741835_1011 (size=395)
2024-12-02T18:31:00,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741835_1011 (size=395)
2024-12-02T18:31:00,918 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b4389760e17f767588510baa116911e9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5
2024-12-02T18:31:00,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41125 is added to blk_1073741836_1012 (size=78)
2024-12-02T18:31:00,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46619 is added to blk_1073741836_1012 (size=78)
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing b4389760e17f767588510baa116911e9, disabling compactions & flushes
2024-12-02T18:31:00,928 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9. after waiting 0 ms
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:00,928 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:00,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for b4389760e17f767588510baa116911e9:
Waiting for close lock at 1733164260928Disabling compacts and flushes for region at 1733164260928Disabling writes for close at 1733164260928Writing region close event to WAL at 1733164260928Closed at 1733164260928
2024-12-02T18:31:00,930 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-02T18:31:00,930 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733164260930"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164260930"}]},"ts":"1733164260930"}
2024-12-02T18:31:00,933 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-02T18:31:00,934 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-02T18:31:00,934 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164260934"}]},"ts":"1733164260934"}
2024-12-02T18:31:00,937 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-02T18:31:00,937 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b4389760e17f767588510baa116911e9, ASSIGN}]
2024-12-02T18:31:00,939 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b4389760e17f767588510baa116911e9, ASSIGN
2024-12-02T18:31:00,940 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b4389760e17f767588510baa116911e9, ASSIGN; state=OFFLINE, location=95451c90bed2,44087,1733164259560; forceNewPlan=false, retain=false
2024-12-02T18:31:01,091 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4389760e17f767588510baa116911e9, regionState=OPENING, regionLocation=95451c90bed2,44087,1733164259560
2024-12-02T18:31:01,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b4389760e17f767588510baa116911e9, ASSIGN because future has completed
2024-12-02T18:31:01,094 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4389760e17f767588510baa116911e9, server=95451c90bed2,44087,1733164259560}]
2024-12-02T18:31:01,252 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:01,253 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b4389760e17f767588510baa116911e9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:31:01,253 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,253 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:01,253 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,253 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,255 INFO  [StoreOpener-b4389760e17f767588510baa116911e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b4389760e17f767588510baa116911e9 
2024-12-02T18:31:01,256 INFO  [StoreOpener-b4389760e17f767588510baa116911e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4389760e17f767588510baa116911e9 columnFamilyName info
2024-12-02T18:31:01,256 DEBUG [StoreOpener-b4389760e17f767588510baa116911e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:01,257 INFO  [StoreOpener-b4389760e17f767588510baa116911e9-1 {}] regionserver.HStore(327): Store=b4389760e17f767588510baa116911e9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:31:01,257 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,258 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,258 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,258 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,258 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,260 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,262 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:31:01,262 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b4389760e17f767588510baa116911e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727467, jitterRate=-0.07497824728488922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:31:01,262 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4389760e17f767588510baa116911e9
2024-12-02T18:31:01,263 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b4389760e17f767588510baa116911e9:
Running coprocessor pre-open hook at 1733164261254Writing region info on filesystem at 1733164261254Initializing all the Stores at 1733164261254Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164261255 (+1 ms)Cleaning up temporary data from old regions at 1733164261258 (+3 ms)Running coprocessor post-open hooks at 1733164261262 (+4 ms)Region opened successfully at 1733164261263 (+1 ms)
2024-12-02T18:31:01,264 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9., pid=6, masterSystemTime=1733164261247
2024-12-02T18:31:01,266 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:01,266 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:01,267 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4389760e17f767588510baa116911e9, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,44087,1733164259560
2024-12-02T18:31:01,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4389760e17f767588510baa116911e9, server=95451c90bed2,44087,1733164259560 because future has completed
2024-12-02T18:31:01,273 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-02T18:31:01,274 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b4389760e17f767588510baa116911e9, server=95451c90bed2,44087,1733164259560 in 177 msec
2024-12-02T18:31:01,276 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-02T18:31:01,276 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b4389760e17f767588510baa116911e9, ASSIGN in 336 msec
2024-12-02T18:31:01,277 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-02T18:31:01,277 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164261277"}]},"ts":"1733164261277"}
2024-12-02T18:31:01,280 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-02T18:31:01,281 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-02T18:31:01,283 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 376 msec
2024-12-02T18:31:01,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:31:01,591 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-02T18:31:01,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-02T18:31:01,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-02T18:31:01,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:31:01,593 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-02T18:31:01,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:01,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:02,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:02,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:03,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:03,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:04,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:04,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:05,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:05,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:05,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,762 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,762 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,762 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:05,763 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,267 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:31:06,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:06,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:31:06,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-02T18:31:06,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:06,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:07,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:07,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:08,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:08,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:09,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:09,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:10,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:10,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:31:10,997 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-02T18:31:10,997 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-02T18:31:11,000 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-02T18:31:11,000 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9., hostname=95451c90bed2,44087,1733164259560, seqNum=2]
2024-12-02T18:31:11,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:11,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:12,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:12,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:13,007 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:13,008 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,008 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,008 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,009 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK], DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]) is bad.
2024-12-02T18:31:13,009 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK], DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]) is bad.
2024-12-02T18:31:13,009 WARN  [PacketResponder: BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41125] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,009 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1114174001_22 at /127.0.0.1:58506 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41125:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58506 dst: /127.0.0.1:41125
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,010 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1114174001_22 at /127.0.0.1:34236 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34236 dst: /127.0.0.1:46619
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,010 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK], DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41125,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]) is bad.
2024-12-02T18:31:13,010 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:34274 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34274 dst: /127.0.0.1:46619
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,010 WARN  [PacketResponder: BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41125] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,010 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:34284 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34284 dst: /127.0.0.1:46619
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:58554 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41125:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58554 dst: /127.0.0.1:41125
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,016 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:58544 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41125:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58544 dst: /127.0.0.1:41125
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,116 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3db1f6e1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:13,116 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1be3b2f7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:13,117 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:13,117 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1118a265{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:13,117 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25cefabc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:13,119 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:13,119 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:13,119 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 25350923-da4e-4b89-8e56-e79264dbe3e6) service to localhost/127.0.0.1:42429
2024-12-02T18:31:13,119 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:13,120 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data3/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:13,120 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data4/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:13,121 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:13,129 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:13,132 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:13,133 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:13,133 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:13,133 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:31:13,134 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57d5f4b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:13,134 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fa662a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:13,240 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@290ac13e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-39619-hadoop-hdfs-3_4_1-tests_jar-_-any-6186299343789690792/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:13,240 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@117b424d{HTTP/1.1, (http/1.1)}{localhost:39619}
2024-12-02T18:31:13,240 INFO  [Time-limited test {}] server.Server(415): Started @168626ms
2024-12-02T18:31:13,242 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:13,270 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,270 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,270 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:13,270 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1114174001_22 at /127.0.0.1:43550 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43550 dst: /127.0.0.1:46619
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,270 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:43526 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43526 dst: /127.0.0.1:46619
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,270 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:43528 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43528 dst: /127.0.0.1:46619
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:13,271 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1935e32{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:13,272 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@31e2f2e1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:13,272 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:13,272 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34c1099f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:13,272 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@623a52f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:13,273 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:13,273 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:13,273 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 48d8bc36-d684-475a-8f7d-706ffcd13c0c) service to localhost/127.0.0.1:42429
2024-12-02T18:31:13,273 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:13,273 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data1/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:13,274 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data2/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:13,274 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:13,281 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:13,285 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:13,286 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:13,286 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:13,286 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:31:13,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64685bd7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:13,287 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20466b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:13,386 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3172a7c6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-40477-hadoop-hdfs-3_4_1-tests_jar-_-any-12724431857883615451/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:13,387 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b559376{HTTP/1.1, (http/1.1)}{localhost:40477}
2024-12-02T18:31:13,387 INFO  [Time-limited test {}] server.Server(415): Started @168773ms
2024-12-02T18:31:13,388 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:13,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:13,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:13,783 WARN  [Thread-1323 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:13,785 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcacc4fa98dc087b7 with lease ID 0xae4f8124982e2cba: from storage DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c node DatanodeRegistration(127.0.0.1:45575, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=39651, infoSecurePort=0, ipcPort=32921, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:13,785 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcacc4fa98dc087b7 with lease ID 0xae4f8124982e2cba: from storage DS-85008a57-6736-4798-9203-383b979f7cc1 node DatanodeRegistration(127.0.0.1:45575, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=39651, infoSecurePort=0, ipcPort=32921, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:13,982 WARN  [Thread-1343 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:13,984 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea13172f9a4885fc with lease ID 0xae4f8124982e2cbb: from storage DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4 node DatanodeRegistration(127.0.0.1:38421, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=45607, infoSecurePort=0, ipcPort=37113, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:13,985 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea13172f9a4885fc with lease ID 0xae4f8124982e2cbb: from storage DS-0c3436e5-7660-4d63-8a7c-d93b2cdc504c node DatanodeRegistration(127.0.0.1:38421, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=45607, infoSecurePort=0, ipcPort=37113, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:14,425 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-02T18:31:14,428 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-02T18:31:14,429 ERROR [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:14,429 WARN  [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:14,430 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C44087%2C1733164259560:(num 1733164260187) roll requested
2024-12-02T18:31:14,430 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:14,436 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 newFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:14,436 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:14,436 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:14,436 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:14,436 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:14,436 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:14,436 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:14,437 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:14,437 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:14,437 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:14,437 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45607:45607),(127.0.0.1/127.0.0.1:39651:39651)]
2024-12-02T18:31:14,437 WARN  [IPC Server handler 0 on default port 42429 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-12-02T18:31:14,437 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 is not closed yet, will try archiving it next time
2024-12-02T18:31:14,438 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 after 1ms
2024-12-02T18:31:14,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:14,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:15,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:15,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:15,785 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-02T18:31:16,446 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-02T18:31:16,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:16,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:17,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:17,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:18,439 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 after 4001ms
2024-12-02T18:31:18,449 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:18,449 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38421,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK], DatanodeInfoWithStorage[127.0.0.1:45575,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38421,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]) is bad.
2024-12-02T18:31:18,450 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:41324 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:45575:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41324 dst: /127.0.0.1:45575
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:18,450 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:41328 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38421:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41328 dst: /127.0.0.1:38421
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:18,546 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3172a7c6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:18,546 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b559376{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:18,546 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:18,546 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20466b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:18,547 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64685bd7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:18,548 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:18,548 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 48d8bc36-d684-475a-8f7d-706ffcd13c0c) service to localhost/127.0.0.1:42429
2024-12-02T18:31:18,548 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:18,548 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:18,548 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data1/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:18,548 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data2/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:18,548 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:18,555 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:18,559 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:18,560 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:18,560 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:18,560 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:31:18,561 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c9115f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:18,561 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ae70be3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:18,678 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@584bc61c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-38255-hadoop-hdfs-3_4_1-tests_jar-_-any-14447417769064367647/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:18,678 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d237fac{HTTP/1.1, (http/1.1)}{localhost:38255}
2024-12-02T18:31:18,679 INFO  [Time-limited test {}] server.Server(415): Started @174065ms
2024-12-02T18:31:18,680 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:18,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:18,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:18,721 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:18,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_640398233_22 at /127.0.0.1:41346 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:45575:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41346 dst: /127.0.0.1:45575
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:18,723 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@290ac13e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:18,724 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@117b424d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:18,724 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:18,724 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fa662a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:18,724 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57d5f4b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:18,725 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:18,725 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:18,725 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 25350923-da4e-4b89-8e56-e79264dbe3e6) service to localhost/127.0.0.1:42429
2024-12-02T18:31:18,725 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:18,725 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data3/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:18,725 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data4/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:18,725 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:18,733 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:18,735 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:18,736 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:18,736 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:18,736 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:31:18,737 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16fecb8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:18,737 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78f99ac1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:18,851 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12a08c73{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/java.io.tmpdir/jetty-localhost-38949-hadoop-hdfs-3_4_1-tests_jar-_-any-1831274461858785893/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:18,852 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7fc558ce{HTTP/1.1, (http/1.1)}{localhost:38949}
2024-12-02T18:31:18,852 INFO  [Time-limited test {}] server.Server(415): Started @174238ms
2024-12-02T18:31:18,853 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:19,123 WARN  [Thread-1397 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:19,125 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc1e21ddfc333379 with lease ID 0xae4f8124982e2cbc: from storage DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4 node DatanodeRegistration(127.0.0.1:37757, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=45903, infoSecurePort=0, ipcPort=45523, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:19,126 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc1e21ddfc333379 with lease ID 0xae4f8124982e2cbc: from storage DS-0c3436e5-7660-4d63-8a7c-d93b2cdc504c node DatanodeRegistration(127.0.0.1:37757, datanodeUuid=48d8bc36-d684-475a-8f7d-706ffcd13c0c, infoPort=45903, infoSecurePort=0, ipcPort=45523, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:19,335 WARN  [Thread-1417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:19,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5befd1e7459fa9 with lease ID 0xae4f8124982e2cbd: from storage DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c node DatanodeRegistration(127.0.0.1:36867, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=39169, infoSecurePort=0, ipcPort=39671, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:19,338 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5befd1e7459fa9 with lease ID 0xae4f8124982e2cbd: from storage DS-85008a57-6736-4798-9203-383b979f7cc1 node DatanodeRegistration(127.0.0.1:36867, datanodeUuid=25350923-da4e-4b89-8e56-e79264dbe3e6, infoPort=39169, infoSecurePort=0, ipcPort=39671, storageInfo=lv=-57;cid=testClusterID;nsid=1639934218;c=1733164256706), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:19,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:19,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:19,886 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-02T18:31:19,888 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-02T18:31:19,889 ERROR [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45575,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:19,889 WARN  [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45575,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:19,889 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C44087%2C1733164259560:(num 1733164274430) roll requested
2024-12-02T18:31:19,890 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:19,895 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 newFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:19,895 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:19,895 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:19,895 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:19,895 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:19,895 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:19,895 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:19,895 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45575,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:19,895 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45575,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:19,896 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:19,896 WARN  [IPC Server handler 3 on default port 42429 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-02T18:31:19,896 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45903:45903),(127.0.0.1/127.0.0.1:39169:39169)]
2024-12-02T18:31:19,896 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 is not closed yet, will try archiving it next time
2024-12-02T18:31:19,896 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 after 0ms
2024-12-02T18:31:20,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:20,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:21,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:21,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:21,897 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:21,902 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890 newFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:21,902 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:21,903 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:21,903 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:21,903 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:21,903 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:21,903 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:21,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39169:39169),(127.0.0.1/127.0.0.1:45903:45903)]
2024-12-02T18:31:21,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 is not closed yet, will try archiving it next time
2024-12-02T18:31:21,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890 is not closed yet, will try archiving it next time
2024-12-02T18:31:21,904 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:21,904 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:21,905 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187 after 1ms
2024-12-02T18:31:21,905 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:21,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741838_1019 (size=1264)
2024-12-02T18:31:21,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741838_1019 (size=1264)
2024-12-02T18:31:21,906 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 is not closed yet, will try archiving it next time
2024-12-02T18:31:21,918 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733164261263/Put/vlen=218/seqid=0]
2024-12-02T18:31:21,918 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733164271005/Put/vlen=1045/seqid=0]
2024-12-02T18:31:21,918 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164260187
2024-12-02T18:31:21,918 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:21,918 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:21,919 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 after 1ms
2024-12-02T18:31:21,919 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:21,922 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733164274429/Put/vlen=1045/seqid=0]
2024-12-02T18:31:21,922 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733164276447/Put/vlen=1045/seqid=0]
2024-12-02T18:31:21,923 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430
2024-12-02T18:31:21,923 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:21,923 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:21,923 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890 after 0ms
2024-12-02T18:31:21,923 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164279890
2024-12-02T18:31:21,927 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733164279889/Put/vlen=1045/seqid=0]
2024-12-02T18:31:21,927 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:21,927 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:21,928 WARN  [IPC Server handler 2 on default port 42429 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-02T18:31:21,928 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 after 1ms
2024-12-02T18:31:22,125 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-02T18:31:22,339 WARN  [ResponseProcessor for block BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:22,339 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1114174001_22 at /127.0.0.1:41750 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36867:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41750 dst: /127.0.0.1:36867
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36867 remote=/127.0.0.1:41750]. Total timeout mills is 60000, 59563 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:22,339 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1114174001_22 at /127.0.0.1:46088 [Receiving block BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37757:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46088 dst: /127.0.0.1:37757
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:22,339 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 block BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36867,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK], DatanodeInfoWithStorage[127.0.0.1:37757,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36867,DS-7321f8bd-83a0-4b82-a1c2-408cfd56256c,DISK]) is bad.
2024-12-02T18:31:22,340 WARN  [DataStreamer for file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 block BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:22,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741839_1022 (size=85)
2024-12-02T18:31:22,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741839_1022 (size=85)
2024-12-02T18:31:22,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:22,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:23,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:23,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:23,898 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164274430 after 4001ms
2024-12-02T18:31:24,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:24,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:25,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:25,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:25,929 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 after 4002ms
2024-12-02T18:31:25,929 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:25,934 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:25,934 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing b4389760e17f767588510baa116911e9 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-02T18:31:25,935 ERROR [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,935 WARN  [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,936 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C44087%2C1733164259560:(num 1733164281897) roll requested
2024-12-02T18:31:25,936 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.1733164285936
2024-12-02T18:31:25,943 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 newFile=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164285936
2024-12-02T18:31:25,943 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,944 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,944 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,944 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,944 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,944 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164285936
2024-12-02T18:31:25,944 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,944 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2118072695-172.17.0.3-1733164256706:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,945 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:25,945 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45903:45903),(127.0.0.1/127.0.0.1:39169:39169)]
2024-12-02T18:31:25,945 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 is not closed yet, will try archiving it next time
2024-12-02T18:31:25,946 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 after 1ms
2024-12-02T18:31:25,952 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.1733164281897 to hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs/95451c90bed2%2C44087%2C1733164259560.1733164281897
2024-12-02T18:31:25,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/.tmp/info/2bbab02d8fb942109d40a1ad469ef4c1 is 1080, key is row1002/info:/1733164271005/Put/seqid=0
2024-12-02T18:31:25,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741841_1024 (size=9270)
2024-12-02T18:31:25,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741841_1024 (size=9270)
2024-12-02T18:31:25,969 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/.tmp/info/2bbab02d8fb942109d40a1ad469ef4c1
2024-12-02T18:31:25,975 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/.tmp/info/2bbab02d8fb942109d40a1ad469ef4c1 as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/info/2bbab02d8fb942109d40a1ad469ef4c1
2024-12-02T18:31:25,981 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/info/2bbab02d8fb942109d40a1ad469ef4c1, entries=4, sequenceid=8, filesize=9.1 K
2024-12-02T18:31:25,982 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for b4389760e17f767588510baa116911e9 in 48ms, sequenceid=8, compaction requested=false
2024-12-02T18:31:25,982 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b4389760e17f767588510baa116911e9:

2024-12-02T18:31:25,983 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-02T18:31:25,983 ERROR [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,983 WARN  [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5-prefix:95451c90bed2,44087,1733164259560.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,983 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C44087%2C1733164259560.meta:.meta(num 1733164260735) roll requested
2024-12-02T18:31:25,983 INFO  [regionserver/95451c90bed2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C44087%2C1733164259560.meta.1733164285983.meta
2024-12-02T18:31:25,992 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,992 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,992 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,992 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,992 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:25,992 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164285983.meta
2024-12-02T18:31:25,993 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,993 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:25,993 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta
2024-12-02T18:31:25,993 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39169:39169),(127.0.0.1/127.0.0.1:45903:45903)]
2024-12-02T18:31:25,993 WARN  [IPC Server handler 1 on default port 42429 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1013
2024-12-02T18:31:25,993 DEBUG [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta is not closed yet, will try archiving it next time
2024-12-02T18:31:25,994 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta after 1ms
2024-12-02T18:31:26,013 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/info/94f68b7463be4893aa8e3ed75dba10c3 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9./info:regioninfo/1733164261267/Put/seqid=0
2024-12-02T18:31:26,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741843_1027 (size=7125)
2024-12-02T18:31:26,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741843_1027 (size=7125)
2024-12-02T18:31:26,022 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/info/94f68b7463be4893aa8e3ed75dba10c3
2024-12-02T18:31:26,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/ns/1c065dd391084d7da1c63bffd5ead840 is 43, key is default/ns:d/1733164260823/Put/seqid=0
2024-12-02T18:31:26,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741844_1028 (size=5153)
2024-12-02T18:31:26,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741844_1028 (size=5153)
2024-12-02T18:31:26,046 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/ns/1c065dd391084d7da1c63bffd5ead840
2024-12-02T18:31:26,067 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/table/fbe5b5f774c74752ba8a3d63d034119b is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733164261277/Put/seqid=0
2024-12-02T18:31:26,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741845_1029 (size=5438)
2024-12-02T18:31:26,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741845_1029 (size=5438)
2024-12-02T18:31:26,072 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/table/fbe5b5f774c74752ba8a3d63d034119b
2024-12-02T18:31:26,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/info/94f68b7463be4893aa8e3ed75dba10c3 as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/info/94f68b7463be4893aa8e3ed75dba10c3
2024-12-02T18:31:26,084 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/info/94f68b7463be4893aa8e3ed75dba10c3, entries=10, sequenceid=11, filesize=7.0 K
2024-12-02T18:31:26,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/ns/1c065dd391084d7da1c63bffd5ead840 as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/ns/1c065dd391084d7da1c63bffd5ead840
2024-12-02T18:31:26,090 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/ns/1c065dd391084d7da1c63bffd5ead840, entries=2, sequenceid=11, filesize=5.0 K
2024-12-02T18:31:26,091 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/.tmp/table/fbe5b5f774c74752ba8a3d63d034119b as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/table/fbe5b5f774c74752ba8a3d63d034119b
2024-12-02T18:31:26,097 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/table/fbe5b5f774c74752ba8a3d63d034119b, entries=2, sequenceid=11, filesize=5.3 K
2024-12-02T18:31:26,098 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=11, compaction requested=false
2024-12-02T18:31:26,099 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-02T18:31:26,104 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:31:26,104 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:31:26,104 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:31:26,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:26,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:26,105 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:31:26,105 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:31:26,105 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=492492848, stopped=false
2024-12-02T18:31:26,105 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,39959,1733164259406
2024-12-02T18:31:26,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:31:26,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:31:26,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:26,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:26,146 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:31:26,147 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:31:26,147 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:31:26,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:26,147 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,44087,1733164259560' *****
2024-12-02T18:31:26,147 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:31:26,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:31:26,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:31:26,148 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:31:26,148 INFO  [RS:0;95451c90bed2:44087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:31:26,148 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:31:26,148 INFO  [RS:0;95451c90bed2:44087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:31:26,148 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(3091): Received CLOSE for b4389760e17f767588510baa116911e9
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,44087,1733164259560
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:44087.
2024-12-02T18:31:26,149 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4389760e17f767588510baa116911e9, disabling compactions & flushes
2024-12-02T18:31:26,149 DEBUG [RS:0;95451c90bed2:44087 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:31:26,149 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:26,149 DEBUG [RS:0;95451c90bed2:44087 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:26,149 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:26,149 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9. after waiting 0 ms
2024-12-02T18:31:26,149 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:31:26,149 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:31:26,150 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-02T18:31:26,150 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1325): Online Regions={b4389760e17f767588510baa116911e9=TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9., 1588230740=hbase:meta,,1.1588230740}
2024-12-02T18:31:26,150 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:31:26,150 DEBUG [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b4389760e17f767588510baa116911e9
2024-12-02T18:31:26,150 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:31:26,150 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:31:26,150 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:31:26,150 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:31:26,155 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/default/TestLogRolling-testLogRollOnPipelineRestart/b4389760e17f767588510baa116911e9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-02T18:31:26,156 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:26,156 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-02T18:31:26,156 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4389760e17f767588510baa116911e9:
Waiting for close lock at 1733164286149Running coprocessor pre-close hooks at 1733164286149Disabling compacts and flushes for region at 1733164286149Disabling writes for close at 1733164286149Writing region close event to WAL at 1733164286150 (+1 ms)Running coprocessor post-close hooks at 1733164286155 (+5 ms)Closed at 1733164286156 (+1 ms)
2024-12-02T18:31:26,156 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733164260904.b4389760e17f767588510baa116911e9.
2024-12-02T18:31:26,156 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:31:26,156 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:31:26,157 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164286150Running coprocessor pre-close hooks at 1733164286150Disabling compacts and flushes for region at 1733164286150Disabling writes for close at 1733164286150Writing region close event to WAL at 1733164286152 (+2 ms)Running coprocessor post-close hooks at 1733164286156 (+4 ms)Closed at 1733164286156
2024-12-02T18:31:26,157 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:31:26,350 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,44087,1733164259560; all regions closed.
2024-12-02T18:31:26,351 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:26,351 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:26,351 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:26,351 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:26,351 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:26,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741842_1025 (size=825)
2024-12-02T18:31:26,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741842_1025 (size=825)
2024-12-02T18:31:26,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:26,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:27,040 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-02T18:31:27,040 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-02T18:31:27,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:27,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:28,045 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:31:28,339 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-02T18:31:28,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:28,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:29,387 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:31:29,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:29,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:29,995 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta after 4001ms
2024-12-02T18:31:29,995 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/WALs/95451c90bed2,44087,1733164259560/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta to hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs/95451c90bed2%2C44087%2C1733164259560.meta.1733164260735.meta
2024-12-02T18:31:29,999 DEBUG [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs
2024-12-02T18:31:29,999 INFO  [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C44087%2C1733164259560.meta:.meta(num 1733164285983)
2024-12-02T18:31:30,000 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,000 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,000 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,000 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,001 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741840_1023 (size=1162)
2024-12-02T18:31:30,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741840_1023 (size=1162)
2024-12-02T18:31:30,008 DEBUG [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs
2024-12-02T18:31:30,008 INFO  [RS:0;95451c90bed2:44087 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C44087%2C1733164259560:(num 1733164285936)
2024-12-02T18:31:30,008 DEBUG [RS:0;95451c90bed2:44087 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:30,008 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:31:30,008 INFO  [RS:0;95451c90bed2:44087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:31:30,008 INFO  [RS:0;95451c90bed2:44087 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:31:30,008 INFO  [RS:0;95451c90bed2:44087 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:31:30,008 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:31:30,009 INFO  [RS:0;95451c90bed2:44087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44087
2024-12-02T18:31:30,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:31:30,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,44087,1733164259560
2024-12-02T18:31:30,062 INFO  [RS:0;95451c90bed2:44087 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:31:30,062 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,44087,1733164259560]
2024-12-02T18:31:30,083 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,44087,1733164259560 already deleted, retry=false
2024-12-02T18:31:30,083 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,44087,1733164259560 expired; onlineServers=0
2024-12-02T18:31:30,083 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,39959,1733164259406' *****
2024-12-02T18:31:30,083 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:31:30,083 INFO  [M:0;95451c90bed2:39959 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:31:30,083 INFO  [M:0;95451c90bed2:39959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:31:30,083 DEBUG [M:0;95451c90bed2:39959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:31:30,083 DEBUG [M:0;95451c90bed2:39959 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:31:30,083 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:31:30,083 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164259915 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164259915,5,FailOnTimeoutGroup]
2024-12-02T18:31:30,083 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164259914 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164259914,5,FailOnTimeoutGroup]
2024-12-02T18:31:30,084 INFO  [M:0;95451c90bed2:39959 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:31:30,084 INFO  [M:0;95451c90bed2:39959 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:31:30,084 DEBUG [M:0;95451c90bed2:39959 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:31:30,084 INFO  [M:0;95451c90bed2:39959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:31:30,084 INFO  [M:0;95451c90bed2:39959 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:31:30,084 INFO  [M:0;95451c90bed2:39959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:31:30,085 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:31:30,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:31:30,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:30,094 DEBUG [M:0;95451c90bed2:39959 {}] zookeeper.ZKUtil(347): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:31:30,094 WARN  [M:0;95451c90bed2:39959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:31:30,094 INFO  [M:0;95451c90bed2:39959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/.lastflushedseqids
2024-12-02T18:31:30,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741846_1030 (size=130)
2024-12-02T18:31:30,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741846_1030 (size=130)
2024-12-02T18:31:30,101 INFO  [M:0;95451c90bed2:39959 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:31:30,101 INFO  [M:0;95451c90bed2:39959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:31:30,101 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:31:30,101 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:30,101 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:30,101 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:31:30,101 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:30,101 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.16 KB heapSize=29.13 KB
2024-12-02T18:31:30,102 ERROR [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData-prefix:95451c90bed2,39959,1733164259406 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:30,102 WARN  [FSHLog-0-hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData-prefix:95451c90bed2,39959,1733164259406 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:30,102 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 95451c90bed2%2C39959%2C1733164259406:(num 1733164259697) roll requested
2024-12-02T18:31:30,102 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C39959%2C1733164259406.1733164290102
2024-12-02T18:31:30,108 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,108 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,108 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,108 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,109 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,109 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 with entries=53, filesize=26.61 KB; new WAL /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164290102
2024-12-02T18:31:30,115 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:30,116 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46619,DS-6e6fac4b-0073-4b0d-bf5c-0ef4947746d4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-02T18:31:30,116 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697
2024-12-02T18:31:30,116 WARN  [IPC Server handler 0 on default port 42429 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-12-02T18:31:30,116 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 after 0ms
2024-12-02T18:31:30,118 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45903:45903),(127.0.0.1/127.0.0.1:39169:39169)]
2024-12-02T18:31:30,118 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 is not closed yet, will try archiving it next time
2024-12-02T18:31:30,133 DEBUG [M:0;95451c90bed2:39959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe2855e5caaf4fc6a1f6955b3a4cc1e8 is 82, key is hbase:meta,,1/info:regioninfo/1733164260766/Put/seqid=0
2024-12-02T18:31:30,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741848_1033 (size=5672)
2024-12-02T18:31:30,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741848_1033 (size=5672)
2024-12-02T18:31:30,138 INFO  [M:0;95451c90bed2:39959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe2855e5caaf4fc6a1f6955b3a4cc1e8
2024-12-02T18:31:30,159 DEBUG [M:0;95451c90bed2:39959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30ff85432e5e4fe88cdc4e8ae76e7e9f is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733164261282/Put/seqid=0
2024-12-02T18:31:30,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741849_1034 (size=6117)
2024-12-02T18:31:30,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741849_1034 (size=6117)
2024-12-02T18:31:30,164 INFO  [M:0;95451c90bed2:39959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.56 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30ff85432e5e4fe88cdc4e8ae76e7e9f
2024-12-02T18:31:30,172 INFO  [RS:0;95451c90bed2:44087 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:31:30,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:31:30,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44087-0x101974fd3e80001, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:31:30,172 INFO  [RS:0;95451c90bed2:44087 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,44087,1733164259560; zookeeper connection closed.
2024-12-02T18:31:30,173 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@384a7d50 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@384a7d50
2024-12-02T18:31:30,173 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:31:30,187 DEBUG [M:0;95451c90bed2:39959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d53c88c18eb4115b87bfb9d5418669c is 69, key is 95451c90bed2,44087,1733164259560/rs:state/1733164260018/Put/seqid=0
2024-12-02T18:31:30,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741850_1035 (size=5156)
2024-12-02T18:31:30,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741850_1035 (size=5156)
2024-12-02T18:31:30,192 INFO  [M:0;95451c90bed2:39959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d53c88c18eb4115b87bfb9d5418669c
2024-12-02T18:31:30,212 DEBUG [M:0;95451c90bed2:39959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b1b40ba9b37486f86f7164ebf955ff0 is 52, key is load_balancer_on/state:d/1733164260899/Put/seqid=0
2024-12-02T18:31:30,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741851_1036 (size=5056)
2024-12-02T18:31:30,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741851_1036 (size=5056)
2024-12-02T18:31:30,219 INFO  [M:0;95451c90bed2:39959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b1b40ba9b37486f86f7164ebf955ff0
2024-12-02T18:31:30,225 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe2855e5caaf4fc6a1f6955b3a4cc1e8 as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe2855e5caaf4fc6a1f6955b3a4cc1e8
2024-12-02T18:31:30,229 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe2855e5caaf4fc6a1f6955b3a4cc1e8, entries=8, sequenceid=56, filesize=5.5 K
2024-12-02T18:31:30,230 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30ff85432e5e4fe88cdc4e8ae76e7e9f as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30ff85432e5e4fe88cdc4e8ae76e7e9f
2024-12-02T18:31:30,237 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30ff85432e5e4fe88cdc4e8ae76e7e9f, entries=6, sequenceid=56, filesize=6.0 K
2024-12-02T18:31:30,238 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7d53c88c18eb4115b87bfb9d5418669c as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7d53c88c18eb4115b87bfb9d5418669c
2024-12-02T18:31:30,244 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7d53c88c18eb4115b87bfb9d5418669c, entries=1, sequenceid=56, filesize=5.0 K
2024-12-02T18:31:30,245 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b1b40ba9b37486f86f7164ebf955ff0 as hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8b1b40ba9b37486f86f7164ebf955ff0
2024-12-02T18:31:30,251 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8b1b40ba9b37486f86f7164ebf955ff0, entries=1, sequenceid=56, filesize=4.9 K
2024-12-02T18:31:30,252 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=56, compaction requested=false
2024-12-02T18:31:30,253 INFO  [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:30,254 DEBUG [M:0;95451c90bed2:39959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164290101Disabling compacts and flushes for region at 1733164290101Disabling writes for close at 1733164290101Obtaining lock to block concurrent updates at 1733164290101Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164290101Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23714, getHeapSize=29768, getOffHeapSize=0, getCellsCount=67 at 1733164290102 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164290119 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164290119Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164290133 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164290133Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164290144 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164290159 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164290159Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164290169 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164290186 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164290186Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164290198 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164290211 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164290211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba779dc: reopening flushed file at 1733164290224 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55ceb442: reopening flushed file at 1733164290230 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b2883f7: reopening flushed file at 1733164290237 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a9479d6: reopening flushed file at 1733164290244 (+7 ms)Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=56, compaction requested=false at 1733164290252 (+8 ms)Writing region close event to WAL at 1733164290253 (+1 ms)Closed at 1733164290253
2024-12-02T18:31:30,254 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,254 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,254 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,254 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,254 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:31:30,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36867 is added to blk_1073741847_1031 (size=757)
2024-12-02T18:31:30,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37757 is added to blk_1073741847_1031 (size=757)
2024-12-02T18:31:30,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:30,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:31,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,181 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,182 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,182 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,185 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,191 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,338 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-02T18:31:31,693 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:31:31,694 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,695 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,695 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,695 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:31,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:31,717 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,718 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,718 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:31,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:31:31,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:31:31,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:31:31,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-02T18:31:32,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:32,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:33,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:33,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:34,117 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 after 4001ms
2024-12-02T18:31:34,118 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/WALs/95451c90bed2,39959,1733164259406/95451c90bed2%2C39959%2C1733164259406.1733164259697 to hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/oldWALs/95451c90bed2%2C39959%2C1733164259406.1733164259697
2024-12-02T18:31:34,121 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/MasterData/oldWALs/95451c90bed2%2C39959%2C1733164259406.1733164259697 to hdfs://localhost:42429/user/jenkins/test-data/60bc6474-aedc-fd1c-40d2-4b64ae5846b5/oldWALs/95451c90bed2%2C39959%2C1733164259406.1733164259697$masterlocalwal$
2024-12-02T18:31:34,121 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:31:34,121 INFO  [M:0;95451c90bed2:39959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:31:34,121 INFO  [M:0;95451c90bed2:39959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39959
2024-12-02T18:31:34,121 INFO  [M:0;95451c90bed2:39959 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:31:34,277 INFO  [M:0;95451c90bed2:39959 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:31:34,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:31:34,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39959-0x101974fd3e80000, quorum=127.0.0.1:64860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:31:34,280 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12a08c73{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:34,281 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7fc558ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:34,281 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:34,281 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78f99ac1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:34,281 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16fecb8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:34,282 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:34,282 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:34,283 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:34,283 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 25350923-da4e-4b89-8e56-e79264dbe3e6) service to localhost/127.0.0.1:42429
2024-12-02T18:31:34,283 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data3/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:34,284 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data4/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:34,284 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:34,287 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@584bc61c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:34,287 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d237fac{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:34,287 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:34,287 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ae70be3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:34,288 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c9115f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:34,289 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:31:34,289 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:31:34,289 WARN  [BP-2118072695-172.17.0.3-1733164256706 heartbeating to localhost/127.0.0.1:42429 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2118072695-172.17.0.3-1733164256706 (Datanode Uuid 48d8bc36-d684-475a-8f7d-706ffcd13c0c) service to localhost/127.0.0.1:42429
2024-12-02T18:31:34,289 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:31:34,290 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data1/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:34,290 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/cluster_8616e747-d654-166c-18e3-c12dc16037bf/data/data2/current/BP-2118072695-172.17.0.3-1733164256706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:31:34,290 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:31:34,296 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@650740c5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:31:34,296 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a99c341{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:31:34,296 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:31:34,297 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37ba1ac4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:31:34,297 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40c321ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir/,STOPPED}
2024-12-02T18:31:34,303 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:31:34,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:31:34,333 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154)
Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42429
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:42429
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42429
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42429
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42429 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42429
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42429 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42429 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 242) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4122 (was 4317)
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=4122
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.log.dir so I do NOT create it in target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b0d802a9-a580-13d8-63a3-36c8ba007d6c/hadoop.tmp.dir so I do NOT create it in target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe, deleteOnExit=true
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/test.cache.data in system properties and HBase conf
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:31:34,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:31:34,342 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:31:34,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:31:34,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:31:34,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:31:34,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:31:34,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:31:34,355 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:31:34,682 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:34,686 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:34,695 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:34,696 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:34,696 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:31:34,697 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:34,697 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:34,698 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:34,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:34,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:34,824 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43498b11{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/java.io.tmpdir/jetty-localhost-38591-hadoop-hdfs-3_4_1-tests_jar-_-any-9694186159992489427/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:31:34,826 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:38591}
2024-12-02T18:31:34,826 INFO  [Time-limited test {}] server.Server(415): Started @190212ms
2024-12-02T18:31:34,842 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:31:35,122 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:35,128 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:35,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:35,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:35,129 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:31:35,130 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4faae08f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:35,130 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c13156e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:35,242 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40527499{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/java.io.tmpdir/jetty-localhost-32867-hadoop-hdfs-3_4_1-tests_jar-_-any-15851987906995141891/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:35,242 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1ed8d4{HTTP/1.1, (http/1.1)}{localhost:32867}
2024-12-02T18:31:35,242 INFO  [Time-limited test {}] server.Server(415): Started @190628ms
2024-12-02T18:31:35,243 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:35,303 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:31:35,307 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:31:35,309 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:31:35,309 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:31:35,309 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:31:35,310 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34534d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:31:35,310 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14a6d451{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:31:35,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c734161{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/java.io.tmpdir/jetty-localhost-33849-hadoop-hdfs-3_4_1-tests_jar-_-any-16235528660900413803/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:31:35,429 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75e789f0{HTTP/1.1, (http/1.1)}{localhost:33849}
2024-12-02T18:31:35,429 INFO  [Time-limited test {}] server.Server(415): Started @190815ms
2024-12-02T18:31:35,430 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:31:35,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:35,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:36,248 WARN  [Thread-1637 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data1/current/BP-2030169084-172.17.0.3-1733164294364/current, will proceed with Du for space computation calculation, 
2024-12-02T18:31:36,249 WARN  [Thread-1638 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data2/current/BP-2030169084-172.17.0.3-1733164294364/current, will proceed with Du for space computation calculation, 
2024-12-02T18:31:36,267 WARN  [Thread-1601 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:36,269 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c745c4c0f49ace8 with lease ID 0xe51356b25b29541c: Processing first storage report for DS-8c6e1900-e2fc-4c24-9859-b03c23bd41db from datanode DatanodeRegistration(127.0.0.1:41397, datanodeUuid=46f9f76c-5e8d-43a0-b072-c9fbf4c45e9a, infoPort=38565, infoSecurePort=0, ipcPort=34101, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364)
2024-12-02T18:31:36,269 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c745c4c0f49ace8 with lease ID 0xe51356b25b29541c: from storage DS-8c6e1900-e2fc-4c24-9859-b03c23bd41db node DatanodeRegistration(127.0.0.1:41397, datanodeUuid=46f9f76c-5e8d-43a0-b072-c9fbf4c45e9a, infoPort=38565, infoSecurePort=0, ipcPort=34101, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:36,269 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c745c4c0f49ace8 with lease ID 0xe51356b25b29541c: Processing first storage report for DS-9250486f-155d-4303-884f-8dc95212d0f0 from datanode DatanodeRegistration(127.0.0.1:41397, datanodeUuid=46f9f76c-5e8d-43a0-b072-c9fbf4c45e9a, infoPort=38565, infoSecurePort=0, ipcPort=34101, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364)
2024-12-02T18:31:36,269 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c745c4c0f49ace8 with lease ID 0xe51356b25b29541c: from storage DS-9250486f-155d-4303-884f-8dc95212d0f0 node DatanodeRegistration(127.0.0.1:41397, datanodeUuid=46f9f76c-5e8d-43a0-b072-c9fbf4c45e9a, infoPort=38565, infoSecurePort=0, ipcPort=34101, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:36,457 WARN  [Thread-1648 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data3/current/BP-2030169084-172.17.0.3-1733164294364/current, will proceed with Du for space computation calculation, 
2024-12-02T18:31:36,457 WARN  [Thread-1649 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data4/current/BP-2030169084-172.17.0.3-1733164294364/current, will proceed with Du for space computation calculation, 
2024-12-02T18:31:36,477 WARN  [Thread-1624 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:31:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x433fd76bfcf16173 with lease ID 0xe51356b25b29541d: Processing first storage report for DS-9f496585-9257-4860-be9e-dc275c80ac87 from datanode DatanodeRegistration(127.0.0.1:40575, datanodeUuid=c9bc10d2-4846-482e-b103-8b87a4a84949, infoPort=35683, infoSecurePort=0, ipcPort=37049, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364)
2024-12-02T18:31:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x433fd76bfcf16173 with lease ID 0xe51356b25b29541d: from storage DS-9f496585-9257-4860-be9e-dc275c80ac87 node DatanodeRegistration(127.0.0.1:40575, datanodeUuid=c9bc10d2-4846-482e-b103-8b87a4a84949, infoPort=35683, infoSecurePort=0, ipcPort=37049, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x433fd76bfcf16173 with lease ID 0xe51356b25b29541d: Processing first storage report for DS-30dbae3f-91d8-49ea-98ce-75e958dbcda6 from datanode DatanodeRegistration(127.0.0.1:40575, datanodeUuid=c9bc10d2-4846-482e-b103-8b87a4a84949, infoPort=35683, infoSecurePort=0, ipcPort=37049, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364)
2024-12-02T18:31:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x433fd76bfcf16173 with lease ID 0xe51356b25b29541d: from storage DS-30dbae3f-91d8-49ea-98ce-75e958dbcda6 node DatanodeRegistration(127.0.0.1:40575, datanodeUuid=c9bc10d2-4846-482e-b103-8b87a4a84949, infoPort=35683, infoSecurePort=0, ipcPort=37049, storageInfo=lv=-57;cid=testClusterID;nsid=1740668831;c=1733164294364), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:31:36,559 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626
2024-12-02T18:31:36,582 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/zookeeper_0, clientPort=54662, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:31:36,582 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54662
2024-12-02T18:31:36,583 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,584 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,594 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:31:36,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:31:36,596 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac with version=8
2024-12-02T18:31:36,596 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:31:36,598 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:31:36,599 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46111
2024-12-02T18:31:36,600 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46111 connecting to ZooKeeper ensemble=127.0.0.1:54662
2024-12-02T18:31:36,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:461110x0, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:31:36,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46111-0x101975065300000 connected
2024-12-02T18:31:36,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:36,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:36,740 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,741 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,745 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:31:36,745 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac, hbase.cluster.distributed=false
2024-12-02T18:31:36,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:31:36,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46111
2024-12-02T18:31:36,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46111
2024-12-02T18:31:36,752 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46111
2024-12-02T18:31:36,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46111
2024-12-02T18:31:36,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46111
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:31:36,771 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:31:36,772 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42095
2024-12-02T18:31:36,773 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42095 connecting to ZooKeeper ensemble=127.0.0.1:54662
2024-12-02T18:31:36,774 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,775 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,782 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420950x0, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:31:36,782 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42095-0x101975065300001 connected
2024-12-02T18:31:36,782 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:31:36,783 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:31:36,785 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:31:36,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:31:36,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:31:36,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42095
2024-12-02T18:31:36,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42095
2024-12-02T18:31:36,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42095
2024-12-02T18:31:36,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42095
2024-12-02T18:31:36,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42095
2024-12-02T18:31:36,798 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:46111
2024-12-02T18:31:36,798 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,46111,1733164296598
2024-12-02T18:31:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:31:36,803 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:31:36,803 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,46111,1733164296598
2024-12-02T18:31:36,813 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:31:36,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,813 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,814 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:31:36,814 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,46111,1733164296598 from backup master directory
2024-12-02T18:31:36,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,46111,1733164296598
2024-12-02T18:31:36,824 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:31:36,824 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:31:36,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:31:36,824 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,46111,1733164296598
2024-12-02T18:31:36,828 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/hbase.id] with ID: c3ad2852-102a-4259-9ea4-45a4d6e815c8
2024-12-02T18:31:36,828 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/.tmp/hbase.id
2024-12-02T18:31:36,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:31:36,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:31:36,835 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/.tmp/hbase.id]:[hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/hbase.id]
2024-12-02T18:31:36,846 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:36,846 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:31:36,848 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-02T18:31:36,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,855 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:31:36,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:31:36,863 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:31:36,863 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:31:36,864 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:31:36,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:31:36,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:31:36,873 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store
2024-12-02T18:31:36,879 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:31:36,879 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:31:36,880 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:36,881 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:31:36,881 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:36,881 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:36,881 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:31:36,881 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:36,881 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:31:36,881 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164296881Disabling compacts and flushes for region at 1733164296881Disabling writes for close at 1733164296881Writing region close event to WAL at 1733164296881Closed at 1733164296881
2024-12-02T18:31:36,882 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/.initializing
2024-12-02T18:31:36,882 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/WALs/95451c90bed2,46111,1733164296598
2024-12-02T18:31:36,884 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C46111%2C1733164296598, suffix=, logDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/WALs/95451c90bed2,46111,1733164296598, archiveDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/oldWALs, maxLogs=10
2024-12-02T18:31:36,885 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C46111%2C1733164296598.1733164296885
2024-12-02T18:31:36,891 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/WALs/95451c90bed2,46111,1733164296598/95451c90bed2%2C46111%2C1733164296598.1733164296885
2024-12-02T18:31:36,892 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38565:38565),(127.0.0.1/127.0.0.1:35683:35683)]
2024-12-02T18:31:36,893 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:31:36,893 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:36,893 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,893 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,894 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:31:36,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:31:36,896 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:36,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:36,897 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:31:36,898 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:31:36,898 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:36,898 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:31:36,899 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:31:36,900 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:31:36,900 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:36,900 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:31:36,900 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:31:36,901 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:31:36,902 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:36,902 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:31:36,902 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,903 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,903 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,904 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,905 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,905 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:31:36,906 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:31:36,908 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:31:36,909 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864919, jitterRate=0.09980255365371704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:31:36,910 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164296893Initializing all the Stores at 1733164296894 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164296894Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164296894Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164296894Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164296894Cleaning up temporary data from old regions at 1733164296905 (+11 ms)Region opened successfully at 1733164296909 (+4 ms)
2024-12-02T18:31:36,910 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:31:36,914 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cacc8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:31:36,914 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:31:36,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:31:36,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:31:36,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:31:36,915 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:31:36,916 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:31:36,916 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:31:36,918 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:31:36,919 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:31:36,929 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:31:36,930 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:31:36,930 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:31:36,940 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:31:36,940 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:31:36,941 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:31:36,950 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:31:36,951 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:31:36,961 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:31:36,963 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:31:36,971 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:31:36,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:31:36,982 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:31:36,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,982 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:36,982 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,46111,1733164296598, sessionid=0x101975065300000, setting cluster-up flag (Was=false)
2024-12-02T18:31:37,003 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,035 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:31:37,036 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,46111,1733164296598
2024-12-02T18:31:37,056 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,087 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:31:37,088 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,46111,1733164296598
2024-12-02T18:31:37,090 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:31:37,091 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:37,092 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:31:37,092 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:31:37,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,46111,1733164296598
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:31:37,094 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,095 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164327095
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:31:37,097 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:31:37,097 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:37,097 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:31:37,097 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:31:37,097 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:31:37,097 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:31:37,097 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164297097,5,FailOnTimeoutGroup]
2024-12-02T18:31:37,097 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164297097,5,FailOnTimeoutGroup]
2024-12-02T18:31:37,097 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,098 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:31:37,098 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,098 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,098 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,098 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:31:37,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:31:37,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:31:37,106 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:31:37,107 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac
2024-12-02T18:31:37,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:31:37,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:31:37,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:37,114 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:31:37,115 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:31:37,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,116 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,116 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:31:37,117 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:31:37,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,117 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,117 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:31:37,118 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:31:37,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,118 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,118 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:31:37,119 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:31:37,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:31:37,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740
2024-12-02T18:31:37,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740
2024-12-02T18:31:37,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:31:37,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:31:37,122 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:31:37,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:31:37,125 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:31:37,125 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734785, jitterRate=-0.06567353010177612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164297113Initializing all the Stores at 1733164297114 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297114Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164297114Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297114Cleaning up temporary data from old regions at 1733164297122 (+8 ms)Region opened successfully at 1733164297126 (+4 ms)
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:31:37,126 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:31:37,126 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:31:37,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164297126Disabling compacts and flushes for region at 1733164297126Disabling writes for close at 1733164297126Writing region close event to WAL at 1733164297126Closed at 1733164297126
2024-12-02T18:31:37,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:37,127 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:31:37,127 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:31:37,128 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:31:37,129 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:31:37,190 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(746): ClusterId : c3ad2852-102a-4259-9ea4-45a4d6e815c8
2024-12-02T18:31:37,190 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:31:37,204 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:31:37,204 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:31:37,215 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:31:37,216 DEBUG [RS:0;95451c90bed2:42095 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c02e0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:31:37,228 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:31:37,229 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,231 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,231 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,235 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:42095
2024-12-02T18:31:37,235 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:31:37,235 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:31:37,235 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:31:37,236 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,46111,1733164296598 with port=42095, startcode=1733164296771
2024-12-02T18:31:37,236 DEBUG [RS:0;95451c90bed2:42095 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:31:37,238 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46697, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:31:37,238 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46111 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,238 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46111 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,240 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac
2024-12-02T18:31:37,240 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34333
2024-12-02T18:31:37,240 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:31:37,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:31:37,246 DEBUG [RS:0;95451c90bed2:42095 {}] zookeeper.ZKUtil(111): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,246 WARN  [RS:0;95451c90bed2:42095 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:31:37,246 INFO  [RS:0;95451c90bed2:42095 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:31:37,246 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,246 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,42095,1733164296771]
2024-12-02T18:31:37,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,249 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:31:37,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,253 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:37,259 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:31:37,259 INFO  [RS:0;95451c90bed2:42095 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:31:37,259 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,259 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:31:37,260 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:31:37,260 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,260 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,261 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,261 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:31:37,261 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:31:37,261 DEBUG [RS:0;95451c90bed2:42095 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,261 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,42095,1733164296771-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:31:37,275 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:31:37,275 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,42095,1733164296771-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,275 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,275 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.Replication(171): 95451c90bed2,42095,1733164296771 started
2024-12-02T18:31:37,279 WARN  [95451c90bed2:46111 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:31:37,286 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,286 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,42095,1733164296771, RpcServer on 95451c90bed2/172.17.0.3:42095, sessionid=0x101975065300001
2024-12-02T18:31:37,286 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:31:37,286 DEBUG [RS:0;95451c90bed2:42095 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,286 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,42095,1733164296771'
2024-12-02T18:31:37,286 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:31:37,287 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:31:37,287 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:31:37,287 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:31:37,287 DEBUG [RS:0;95451c90bed2:42095 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,288 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,42095,1733164296771'
2024-12-02T18:31:37,288 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:31:37,288 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:31:37,288 DEBUG [RS:0;95451c90bed2:42095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:31:37,288 INFO  [RS:0;95451c90bed2:42095 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:31:37,288 INFO  [RS:0;95451c90bed2:42095 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:31:37,390 INFO  [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C42095%2C1733164296771, suffix=, logDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771, archiveDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs, maxLogs=32
2024-12-02T18:31:37,391 INFO  [RS:0;95451c90bed2:42095 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42095%2C1733164296771.1733164297391
2024-12-02T18:31:37,397 INFO  [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164297391
2024-12-02T18:31:37,405 DEBUG [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38565:38565),(127.0.0.1/127.0.0.1:35683:35683)]
2024-12-02T18:31:37,530 DEBUG [95451c90bed2:46111 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:31:37,531 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,534 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,42095,1733164296771, state=OPENING
2024-12-02T18:31:37,593 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:31:37,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,604 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:31:37,606 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:31:37,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:37,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:37,606 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,42095,1733164296771}]
2024-12-02T18:31:37,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:37,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:37,763 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:31:37,765 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54675, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:31:37,769 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:31:37,769 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:31:37,772 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C42095%2C1733164296771.meta, suffix=.meta, logDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771, archiveDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs, maxLogs=32
2024-12-02T18:31:37,773 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42095%2C1733164296771.meta.1733164297772.meta
2024-12-02T18:31:37,778 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.meta.1733164297772.meta
2024-12-02T18:31:37,796 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38565:38565),(127.0.0.1/127.0.0.1:35683:35683)]
2024-12-02T18:31:37,809 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:31:37,809 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:31:37,809 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:31:37,809 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:31:37,810 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:31:37,810 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:37,810 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:31:37,810 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:31:37,823 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:31:37,824 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:31:37,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,825 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,825 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:31:37,825 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:31:37,825 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,826 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,826 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:31:37,826 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:31:37,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,827 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,827 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:31:37,827 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:31:37,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:37,828 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:31:37,828 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:31:37,829 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740
2024-12-02T18:31:37,830 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740
2024-12-02T18:31:37,831 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:31:37,831 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:31:37,831 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:31:37,832 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:31:37,833 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767126, jitterRate=-0.024549022316932678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:31:37,833 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:31:37,833 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164297810Writing region info on filesystem at 1733164297810Initializing all the Stores at 1733164297811 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297811Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297823 (+12 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164297823Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164297823Cleaning up temporary data from old regions at 1733164297831 (+8 ms)Running coprocessor post-open hooks at 1733164297833 (+2 ms)Region opened successfully at 1733164297833
2024-12-02T18:31:37,834 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164297762
2024-12-02T18:31:37,837 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:31:37,837 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:31:37,838 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,839 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,42095,1733164296771, state=OPEN
2024-12-02T18:31:37,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:31:37,874 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:31:37,874 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,42095,1733164296771
2024-12-02T18:31:37,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:37,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:31:37,877 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:31:37,877 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,42095,1733164296771 in 268 msec
2024-12-02T18:31:37,880 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:31:37,880 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 750 msec
2024-12-02T18:31:37,882 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:31:37,882 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:31:37,883 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:31:37,883 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,42095,1733164296771, seqNum=-1]
2024-12-02T18:31:37,884 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:31:37,885 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:31:37,890 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 798 msec
2024-12-02T18:31:37,890 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164297890, completionTime=-1
2024-12-02T18:31:37,890 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:31:37,890 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:31:37,892 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:31:37,892 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164357892
2024-12-02T18:31:37,892 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164417892
2024-12-02T18:31:37,892 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:46111, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,893 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,894 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.073sec
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:31:37,897 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:31:37,899 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:31:37,899 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:31:37,899 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,46111,1733164296598-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:31:37,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73db706f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:37,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,46111,-1 for getting cluster id
2024-12-02T18:31:37,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:31:37,993 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c3ad2852-102a-4259-9ea4-45a4d6e815c8'
2024-12-02T18:31:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:31:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c3ad2852-102a-4259-9ea4-45a4d6e815c8"

2024-12-02T18:31:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b78a954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,46111,-1]
2024-12-02T18:31:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:31:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:31:37,996 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:31:37,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae4e6f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:31:37,999 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:31:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,42095,1733164296771, seqNum=-1]
2024-12-02T18:31:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:31:38,002 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:31:38,004 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,46111,1733164296598
2024-12-02T18:31:38,004 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:31:38,007 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:31:38,008 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-02T18:31:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 95451c90bed2,46111,1733164296598
2024-12-02T18:31:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13640fbc
2024-12-02T18:31:38,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-02T18:31:38,010 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-02T18:31:38,011 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-02T18:31:38,011 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-02T18:31:38,011 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:31:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:38,013 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-02T18:31:38,013 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:38,014 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-02T18:31:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:31:38,015 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-02T18:31:38,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741835_1011 (size=405)
2024-12-02T18:31:38,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741835_1011 (size=405)
2024-12-02T18:31:38,024 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 886f80312641ddcc7b2be631407a2fe5, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac
2024-12-02T18:31:38,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741836_1012 (size=88)
2024-12-02T18:31:38,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741836_1012 (size=88)
2024-12-02T18:31:38,031 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:38,031 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 886f80312641ddcc7b2be631407a2fe5, disabling compactions & flushes
2024-12-02T18:31:38,031 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,031 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,031 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5. after waiting 0 ms
2024-12-02T18:31:38,031 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,031 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,032 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 886f80312641ddcc7b2be631407a2fe5:
Waiting for close lock at 1733164298031Disabling compacts and flushes for region at 1733164298031Disabling writes for close at 1733164298031Writing region close event to WAL at 1733164298031Closed at 1733164298031
2024-12-02T18:31:38,033 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-02T18:31:38,033 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733164298033"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164298033"}]},"ts":"1733164298033"}
2024-12-02T18:31:38,036 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-02T18:31:38,037 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-02T18:31:38,037 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164298037"}]},"ts":"1733164298037"}
2024-12-02T18:31:38,039 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-02T18:31:38,039 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=886f80312641ddcc7b2be631407a2fe5, ASSIGN}]
2024-12-02T18:31:38,041 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=886f80312641ddcc7b2be631407a2fe5, ASSIGN
2024-12-02T18:31:38,042 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=886f80312641ddcc7b2be631407a2fe5, ASSIGN; state=OFFLINE, location=95451c90bed2,42095,1733164296771; forceNewPlan=false, retain=false
2024-12-02T18:31:38,193 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=886f80312641ddcc7b2be631407a2fe5, regionState=OPENING, regionLocation=95451c90bed2,42095,1733164296771
2024-12-02T18:31:38,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=886f80312641ddcc7b2be631407a2fe5, ASSIGN because future has completed
2024-12-02T18:31:38,196 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 886f80312641ddcc7b2be631407a2fe5, server=95451c90bed2,42095,1733164296771}]
2024-12-02T18:31:38,359 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,359 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 886f80312641ddcc7b2be631407a2fe5, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:31:38,359 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,359 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:31:38,359 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,359 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,361 INFO  [StoreOpener-886f80312641ddcc7b2be631407a2fe5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 886f80312641ddcc7b2be631407a2fe5 
2024-12-02T18:31:38,362 INFO  [StoreOpener-886f80312641ddcc7b2be631407a2fe5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 886f80312641ddcc7b2be631407a2fe5 columnFamilyName info
2024-12-02T18:31:38,362 DEBUG [StoreOpener-886f80312641ddcc7b2be631407a2fe5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:31:38,362 INFO  [StoreOpener-886f80312641ddcc7b2be631407a2fe5-1 {}] regionserver.HStore(327): Store=886f80312641ddcc7b2be631407a2fe5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:31:38,362 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,363 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,363 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,364 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,364 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,365 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,367 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:31:38,368 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 886f80312641ddcc7b2be631407a2fe5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834209, jitterRate=0.060752227902412415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:31:38,368 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:31:38,368 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 886f80312641ddcc7b2be631407a2fe5:
Running coprocessor pre-open hook at 1733164298360Writing region info on filesystem at 1733164298360Initializing all the Stores at 1733164298360Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164298360Cleaning up temporary data from old regions at 1733164298364 (+4 ms)Running coprocessor post-open hooks at 1733164298368 (+4 ms)Region opened successfully at 1733164298368
2024-12-02T18:31:38,369 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5., pid=6, masterSystemTime=1733164298355
2024-12-02T18:31:38,372 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,372 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:38,373 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=886f80312641ddcc7b2be631407a2fe5, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,42095,1733164296771
2024-12-02T18:31:38,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 886f80312641ddcc7b2be631407a2fe5, server=95451c90bed2,42095,1733164296771 because future has completed
2024-12-02T18:31:38,379 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-02T18:31:38,379 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 886f80312641ddcc7b2be631407a2fe5, server=95451c90bed2,42095,1733164296771 in 181 msec
2024-12-02T18:31:38,382 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-02T18:31:38,382 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=886f80312641ddcc7b2be631407a2fe5, ASSIGN in 340 msec
2024-12-02T18:31:38,383 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-02T18:31:38,384 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164298383"}]},"ts":"1733164298383"}
2024-12-02T18:31:38,386 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-02T18:31:38,387 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-02T18:31:38,389 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 376 msec
2024-12-02T18:31:38,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:38,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:39,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:39,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:40,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:40,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:41,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:31:41,591 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-02T18:31:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:31:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-02T18:31:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:31:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-02T18:31:41,593 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:41,593 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-02T18:31:41,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:41,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:42,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:42,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:43,312 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:31:43,313 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,340 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:31:43,346 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:31:43,346 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-02T18:31:43,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:43,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:44,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:44,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:45,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:45,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:46,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:46,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:47,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:47,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:31:48,036 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-02T18:31:48,037 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-02T18:31:48,041 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:48,041 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5., hostname=95451c90bed2,42095,1733164296771, seqNum=2]
2024-12-02T18:31:48,052 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:48,057 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-02T18:31:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-02T18:31:48,059 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-02T18:31:48,060 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-02T18:31:48,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-02T18:31:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:48,222 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 886f80312641ddcc7b2be631407a2fe5 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-02T18:31:48,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/0a5ed13b72874f3986f35c2126869411 is 1080, key is row0001/info:/1733164308046/Put/seqid=0
2024-12-02T18:31:48,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741837_1013 (size=6033)
2024-12-02T18:31:48,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741837_1013 (size=6033)
2024-12-02T18:31:48,243 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/0a5ed13b72874f3986f35c2126869411
2024-12-02T18:31:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/0a5ed13b72874f3986f35c2126869411 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411
2024-12-02T18:31:48,257 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411, entries=1, sequenceid=5, filesize=5.9 K
2024-12-02T18:31:48,258 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 36ms, sequenceid=5, compaction requested=false
2024-12-02T18:31:48,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 886f80312641ddcc7b2be631407a2fe5:

2024-12-02T18:31:48,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-02T18:31:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-02T18:31:48,266 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-02T18:31:48,266 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 204 msec
2024-12-02T18:31:48,269 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 214 msec
2024-12-02T18:31:48,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:48,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:49,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:49,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:50,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:50,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:51,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:51,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:52,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:52,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:53,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:53,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:54,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:54,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:55,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:55,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:56,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:56,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:57,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:57,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:57,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537 after 68058ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:57,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta after 68045ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-02T18:31:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-02T18:31:58,116 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-02T18:31:58,119 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:31:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-02T18:31:58,123 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-02T18:31:58,124 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-02T18:31:58,124 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-02T18:31:58,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-02T18:31:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:58,280 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 886f80312641ddcc7b2be631407a2fe5 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-02T18:31:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/5ba7df6d30a143f99a3524024a4df643 is 1080, key is row0002/info:/1733164318117/Put/seqid=0
2024-12-02T18:31:58,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741838_1014 (size=6033)
2024-12-02T18:31:58,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741838_1014 (size=6033)
2024-12-02T18:31:58,295 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/5ba7df6d30a143f99a3524024a4df643
2024-12-02T18:31:58,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/5ba7df6d30a143f99a3524024a4df643 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643
2024-12-02T18:31:58,308 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643, entries=1, sequenceid=9, filesize=5.9 K
2024-12-02T18:31:58,309 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 30ms, sequenceid=9, compaction requested=false
2024-12-02T18:31:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 886f80312641ddcc7b2be631407a2fe5:

2024-12-02T18:31:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:31:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-02T18:31:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-02T18:31:58,313 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-02T18:31:58,313 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-12-02T18:31:58,316 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-12-02T18:31:58,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:58,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:59,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:31:59,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:00,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:00,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:01,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:01,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:02,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:02,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:03,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:03,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:04,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:04,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:05,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:05,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:06,558 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:32:06,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:06,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:07,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:07,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-02T18:32:08,176 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-02T18:32:08,180 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42095%2C1733164296771.1733164328180
2024-12-02T18:32:08,186 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:08,186 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:08,186 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:08,186 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:08,186 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:08,186 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164297391 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164328180
2024-12-02T18:32:08,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38565:38565),(127.0.0.1/127.0.0.1:35683:35683)]
2024-12-02T18:32:08,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164297391 is not closed yet, will try archiving it next time
2024-12-02T18:32:08,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741833_1009 (size=5546)
2024-12-02T18:32:08,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741833_1009 (size=5546)
2024-12-02T18:32:08,188 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:32:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:32:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-02T18:32:08,190 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-02T18:32:08,191 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-02T18:32:08,191 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-02T18:32:08,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-02T18:32:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:08,345 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 886f80312641ddcc7b2be631407a2fe5 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-02T18:32:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/14317601a9e14b2aafb6b2d8ab0faeb3 is 1080, key is row0003/info:/1733164328178/Put/seqid=0
2024-12-02T18:32:08,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741840_1016 (size=6033)
2024-12-02T18:32:08,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741840_1016 (size=6033)
2024-12-02T18:32:08,361 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/14317601a9e14b2aafb6b2d8ab0faeb3
2024-12-02T18:32:08,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/14317601a9e14b2aafb6b2d8ab0faeb3 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3
2024-12-02T18:32:08,374 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3, entries=1, sequenceid=13, filesize=5.9 K
2024-12-02T18:32:08,375 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 30ms, sequenceid=13, compaction requested=true
2024-12-02T18:32:08,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 886f80312641ddcc7b2be631407a2fe5:

2024-12-02T18:32:08,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:08,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-02T18:32:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-02T18:32:08,380 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-02T18:32:08,381 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-12-02T18:32:08,384 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 193 msec
2024-12-02T18:32:08,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:08,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:09,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:09,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:10,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:10,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:11,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:11,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:12,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:12,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:13,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:13,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:14,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:14,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:15,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:15,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:16,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:16,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:17,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:17,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-02T18:32:18,206 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-02T18:32:18,206 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:32:18,207 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:32:18,207 DEBUG [Time-limited test {}] regionserver.HStore(1541): 886f80312641ddcc7b2be631407a2fe5/info is initiating minor compaction (all files)
2024-12-02T18:32:18,207 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:32:18,207 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:18,208 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 886f80312641ddcc7b2be631407a2fe5/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:18,208 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3] into tmpdir=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp, totalSize=17.7 K
2024-12-02T18:32:18,208 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0a5ed13b72874f3986f35c2126869411, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733164308046
2024-12-02T18:32:18,208 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5ba7df6d30a143f99a3524024a4df643, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733164318117
2024-12-02T18:32:18,209 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 14317601a9e14b2aafb6b2d8ab0faeb3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733164328178
2024-12-02T18:32:18,221 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 886f80312641ddcc7b2be631407a2fe5#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:32:18,222 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/c1fe75ab8eb845a89368bca8ce1247f0 is 1080, key is row0001/info:/1733164308046/Put/seqid=0
2024-12-02T18:32:18,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741841_1017 (size=8296)
2024-12-02T18:32:18,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741841_1017 (size=8296)
2024-12-02T18:32:18,234 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/c1fe75ab8eb845a89368bca8ce1247f0 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/c1fe75ab8eb845a89368bca8ce1247f0
2024-12-02T18:32:18,242 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 886f80312641ddcc7b2be631407a2fe5/info of 886f80312641ddcc7b2be631407a2fe5 into c1fe75ab8eb845a89368bca8ce1247f0(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:32:18,242 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 886f80312641ddcc7b2be631407a2fe5:

2024-12-02T18:32:18,245 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42095%2C1733164296771.1733164338245
2024-12-02T18:32:18,253 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:18,254 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:18,254 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:18,254 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:18,254 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:18,254 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164328180 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164338245
2024-12-02T18:32:18,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38565:38565),(127.0.0.1/127.0.0.1:35683:35683)]
2024-12-02T18:32:18,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164328180 is not closed yet, will try archiving it next time
2024-12-02T18:32:18,256 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164297391 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs/95451c90bed2%2C42095%2C1733164296771.1733164297391
2024-12-02T18:32:18,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741839_1015 (size=2520)
2024-12-02T18:32:18,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741839_1015 (size=2520)
2024-12-02T18:32:18,257 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:32:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:32:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-02T18:32:18,260 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-02T18:32:18,261 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-02T18:32:18,261 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-02T18:32:18,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42095 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-02T18:32:18,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:18,414 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 886f80312641ddcc7b2be631407a2fe5 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-02T18:32:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4d49856526e24a718703601cdd807fa8 is 1080, key is row0000/info:/1733164338243/Put/seqid=0
2024-12-02T18:32:18,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741843_1019 (size=6033)
2024-12-02T18:32:18,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741843_1019 (size=6033)
2024-12-02T18:32:18,425 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4d49856526e24a718703601cdd807fa8
2024-12-02T18:32:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4d49856526e24a718703601cdd807fa8 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/4d49856526e24a718703601cdd807fa8
2024-12-02T18:32:18,439 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/4d49856526e24a718703601cdd807fa8, entries=1, sequenceid=18, filesize=5.9 K
2024-12-02T18:32:18,440 INFO  [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 26ms, sequenceid=18, compaction requested=false
2024-12-02T18:32:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 886f80312641ddcc7b2be631407a2fe5:

2024-12-02T18:32:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-02T18:32:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-02T18:32:18,444 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-02T18:32:18,444 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 181 msec
2024-12-02T18:32:18,447 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec
2024-12-02T18:32:18,542 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-02T18:32:18,542 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-02T18:32:18,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:18,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:19,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:19,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:20,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:20,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:21,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:21,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:22,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:22,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:23,359 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 886f80312641ddcc7b2be631407a2fe5, had cached 0 bytes from a total of 14329
2024-12-02T18:32:23,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:23,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:24,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:24,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:25,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:25,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:26,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:26,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:27,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:27,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-02T18:32:28,356 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-02T18:32:28,358 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C42095%2C1733164296771.1733164348358
2024-12-02T18:32:28,364 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,364 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,364 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,365 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,365 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,365 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164338245 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164348358
2024-12-02T18:32:28,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35683:35683),(127.0.0.1/127.0.0.1:38565:38565)]
2024-12-02T18:32:28,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164338245 is not closed yet, will try archiving it next time
2024-12-02T18:32:28,366 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164328180 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs/95451c90bed2%2C42095%2C1733164296771.1733164328180
2024-12-02T18:32:28,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:32:28,366 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:32:28,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:32:28,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:28,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:28,367 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:32:28,367 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:32:28,367 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1737506064, stopped=false
2024-12-02T18:32:28,367 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,46111,1733164296598
2024-12-02T18:32:28,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741842_1018 (size=2026)
2024-12-02T18:32:28,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741842_1018 (size=2026)
2024-12-02T18:32:28,453 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:32:28,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:32:28,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:28,453 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:28,453 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:32:28,454 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:32:28,454 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:32:28,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:28,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:32:28,454 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,42095,1733164296771' *****
2024-12-02T18:32:28,454 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:32:28,454 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:32:28,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(3091): Received CLOSE for 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:32:28,455 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,42095,1733164296771
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:32:28,455 INFO  [RS:0;95451c90bed2:42095 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:42095.
2024-12-02T18:32:28,456 DEBUG [RS:0;95451c90bed2:42095 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 886f80312641ddcc7b2be631407a2fe5, disabling compactions & flushes
2024-12-02T18:32:28,456 DEBUG [RS:0;95451c90bed2:42095 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:28,456 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:28,456 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:32:28,456 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:28,456 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:32:28,456 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5. after waiting 0 ms
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:28,456 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-02T18:32:28,456 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 886f80312641ddcc7b2be631407a2fe5 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-02T18:32:28,456 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1325): Online Regions={886f80312641ddcc7b2be631407a2fe5=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5., 1588230740=hbase:meta,,1.1588230740}
2024-12-02T18:32:28,456 DEBUG [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 886f80312641ddcc7b2be631407a2fe5
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:32:28,456 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:32:28,456 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:32:28,457 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-02T18:32:28,462 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4de173ad78e94626ba3efb64bb60f078 is 1080, key is row0001/info:/1733164348357/Put/seqid=0
2024-12-02T18:32:28,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741845_1021 (size=6033)
2024-12-02T18:32:28,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741845_1021 (size=6033)
2024-12-02T18:32:28,467 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4de173ad78e94626ba3efb64bb60f078
2024-12-02T18:32:28,474 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/.tmp/info/4de173ad78e94626ba3efb64bb60f078 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/4de173ad78e94626ba3efb64bb60f078
2024-12-02T18:32:28,475 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/info/a2fb754566b647a08b3907faea15f2d3 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5./info:regioninfo/1733164298372/Put/seqid=0
2024-12-02T18:32:28,480 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/4de173ad78e94626ba3efb64bb60f078, entries=1, sequenceid=22, filesize=5.9 K
2024-12-02T18:32:28,481 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 25ms, sequenceid=22, compaction requested=true
2024-12-02T18:32:28,490 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3] to archive
2024-12-02T18:32:28,491 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:32:28,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741846_1022 (size=7308)
2024-12-02T18:32:28,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741846_1022 (size=7308)
2024-12-02T18:32:28,493 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/0a5ed13b72874f3986f35c2126869411
2024-12-02T18:32:28,494 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/info/a2fb754566b647a08b3907faea15f2d3
2024-12-02T18:32:28,495 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/5ba7df6d30a143f99a3524024a4df643
2024-12-02T18:32:28,496 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/info/14317601a9e14b2aafb6b2d8ab0faeb3
2024-12-02T18:32:28,496 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=95451c90bed2:46111 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-02T18:32:28,496 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [0a5ed13b72874f3986f35c2126869411=6033, 5ba7df6d30a143f99a3524024a4df643=6033, 14317601a9e14b2aafb6b2d8ab0faeb3=6033]
2024-12-02T18:32:28,500 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/886f80312641ddcc7b2be631407a2fe5/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-02T18:32:28,501 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:28,501 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 886f80312641ddcc7b2be631407a2fe5:
Waiting for close lock at 1733164348456Running coprocessor pre-close hooks at 1733164348456Disabling compacts and flushes for region at 1733164348456Disabling writes for close at 1733164348456Obtaining lock to block concurrent updates at 1733164348456Preparing flush snapshotting stores in 886f80312641ddcc7b2be631407a2fe5 at 1733164348456Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733164348457 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5. at 1733164348458 (+1 ms)Flushing 886f80312641ddcc7b2be631407a2fe5/info: creating writer at 1733164348458Flushing 886f80312641ddcc7b2be631407a2fe5/info: appending metadata at 1733164348461 (+3 ms)Flushing 886f80312641ddcc7b2be631407a2fe5/info: closing flushed file at 1733164348461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e6277cd: reopening flushed file at 1733164348473 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 886f80312641ddcc7b2be631407a2fe5 in 25ms, sequenceid=22, compaction requested=true at 1733164348481 (+8 ms)Writing region close event to WAL at 1733164348497 (+16 ms)Running coprocessor post-close hooks at 1733164348501 (+4 ms)Closed at 1733164348501
2024-12-02T18:32:28,501 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733164298011.886f80312641ddcc7b2be631407a2fe5.
2024-12-02T18:32:28,515 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/ns/4b51b82a0706438ca30a6cb5a17268e0 is 43, key is default/ns:d/1733164297885/Put/seqid=0
2024-12-02T18:32:28,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741847_1023 (size=5153)
2024-12-02T18:32:28,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741847_1023 (size=5153)
2024-12-02T18:32:28,520 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/ns/4b51b82a0706438ca30a6cb5a17268e0
2024-12-02T18:32:28,539 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/table/9c8c2f06a0ce410494fea4435ec84d23 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733164298383/Put/seqid=0
2024-12-02T18:32:28,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741848_1024 (size=5508)
2024-12-02T18:32:28,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741848_1024 (size=5508)
2024-12-02T18:32:28,544 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/table/9c8c2f06a0ce410494fea4435ec84d23
2024-12-02T18:32:28,549 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/info/a2fb754566b647a08b3907faea15f2d3 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/info/a2fb754566b647a08b3907faea15f2d3
2024-12-02T18:32:28,555 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/info/a2fb754566b647a08b3907faea15f2d3, entries=10, sequenceid=11, filesize=7.1 K
2024-12-02T18:32:28,557 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/ns/4b51b82a0706438ca30a6cb5a17268e0 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/ns/4b51b82a0706438ca30a6cb5a17268e0
2024-12-02T18:32:28,563 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/ns/4b51b82a0706438ca30a6cb5a17268e0, entries=2, sequenceid=11, filesize=5.0 K
2024-12-02T18:32:28,564 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/.tmp/table/9c8c2f06a0ce410494fea4435ec84d23 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/table/9c8c2f06a0ce410494fea4435ec84d23
2024-12-02T18:32:28,570 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/table/9c8c2f06a0ce410494fea4435ec84d23, entries=2, sequenceid=11, filesize=5.4 K
2024-12-02T18:32:28,571 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false
2024-12-02T18:32:28,580 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-02T18:32:28,580 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:32:28,580 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:32:28,580 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164348456Running coprocessor pre-close hooks at 1733164348456Disabling compacts and flushes for region at 1733164348456Disabling writes for close at 1733164348456Obtaining lock to block concurrent updates at 1733164348457 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733164348457Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733164348457Flushing stores of hbase:meta,,1.1588230740 at 1733164348458 (+1 ms)Flushing 1588230740/info: creating writer at 1733164348458Flushing 1588230740/info: appending metadata at 1733164348475 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733164348475Flushing 1588230740/ns: creating writer at 1733164348499 (+24 ms)Flushing 1588230740/ns: appending metadata at 1733164348514 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733164348514Flushing 1588230740/table: creating writer at 1733164348525 (+11 ms)Flushing 1588230740/table: appending metadata at 1733164348539 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733164348539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531b6d78: reopening flushed file at 1733164348549 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bd58f96: reopening flushed file at 1733164348555 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b0b07d9: reopening flushed file at 1733164348563 (+8 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false at 1733164348571 (+8 ms)Writing region close event to WAL at 1733164348577 (+6 ms)Running coprocessor post-close hooks at 1733164348580 (+3 ms)Closed at 1733164348580
2024-12-02T18:32:28,580 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:32:28,656 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,42095,1733164296771; all regions closed.
2024-12-02T18:32:28,657 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,657 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,657 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,657 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,657 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741834_1010 (size=3306)
2024-12-02T18:32:28,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741834_1010 (size=3306)
2024-12-02T18:32:28,661 DEBUG [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs
2024-12-02T18:32:28,661 INFO  [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C42095%2C1733164296771.meta:.meta(num 1733164297772)
2024-12-02T18:32:28,661 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,661 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,661 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,661 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,661 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:28,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741844_1020 (size=1252)
2024-12-02T18:32:28,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741844_1020 (size=1252)
2024-12-02T18:32:28,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:28,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:28,768 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/WALs/95451c90bed2,42095,1733164296771/95451c90bed2%2C42095%2C1733164296771.1733164338245 to hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs/95451c90bed2%2C42095%2C1733164296771.1733164338245
2024-12-02T18:32:28,770 DEBUG [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/oldWALs
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C42095%2C1733164296771:(num 1733164348358)
2024-12-02T18:32:28,771 DEBUG [RS:0;95451c90bed2:42095 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:32:28,771 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:32:28,771 INFO  [RS:0;95451c90bed2:42095 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42095
2024-12-02T18:32:28,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:32:28,821 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,42095,1733164296771
2024-12-02T18:32:28,821 INFO  [RS:0;95451c90bed2:42095 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:32:28,832 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,42095,1733164296771]
2024-12-02T18:32:28,842 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,42095,1733164296771 already deleted, retry=false
2024-12-02T18:32:28,842 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,42095,1733164296771 expired; onlineServers=0
2024-12-02T18:32:28,842 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,46111,1733164296598' *****
2024-12-02T18:32:28,842 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:32:28,842 INFO  [M:0;95451c90bed2:46111 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:32:28,842 INFO  [M:0;95451c90bed2:46111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:32:28,843 DEBUG [M:0;95451c90bed2:46111 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:32:28,843 DEBUG [M:0;95451c90bed2:46111 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:32:28,843 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:32:28,843 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164297097 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164297097,5,FailOnTimeoutGroup]
2024-12-02T18:32:28,843 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164297097 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164297097,5,FailOnTimeoutGroup]
2024-12-02T18:32:28,843 INFO  [M:0;95451c90bed2:46111 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:32:28,843 INFO  [M:0;95451c90bed2:46111 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:32:28,843 DEBUG [M:0;95451c90bed2:46111 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:32:28,843 INFO  [M:0;95451c90bed2:46111 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:32:28,843 INFO  [M:0;95451c90bed2:46111 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:32:28,843 INFO  [M:0;95451c90bed2:46111 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:32:28,844 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:32:28,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:32:28,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:28,853 DEBUG [M:0;95451c90bed2:46111 {}] zookeeper.ZKUtil(347): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:32:28,853 WARN  [M:0;95451c90bed2:46111 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:32:28,854 INFO  [M:0;95451c90bed2:46111 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/.lastflushedseqids
2024-12-02T18:32:28,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741849_1025 (size=130)
2024-12-02T18:32:28,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741849_1025 (size=130)
2024-12-02T18:32:28,860 INFO  [M:0;95451c90bed2:46111 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:32:28,861 INFO  [M:0;95451c90bed2:46111 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:32:28,861 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:32:28,861 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:28,861 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:28,861 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:32:28,861 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:28,861 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.54 KB heapSize=54.91 KB
2024-12-02T18:32:28,876 DEBUG [M:0;95451c90bed2:46111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a95a1d22d4149ebb7e84055c1da234c is 82, key is hbase:meta,,1/info:regioninfo/1733164297838/Put/seqid=0
2024-12-02T18:32:28,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741850_1026 (size=5672)
2024-12-02T18:32:28,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741850_1026 (size=5672)
2024-12-02T18:32:28,881 INFO  [M:0;95451c90bed2:46111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a95a1d22d4149ebb7e84055c1da234c
2024-12-02T18:32:28,902 DEBUG [M:0;95451c90bed2:46111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3ca8f92bdc346d7946d0ddab0d557f5 is 797, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733164298388/Put/seqid=0
2024-12-02T18:32:28,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741851_1027 (size=7818)
2024-12-02T18:32:28,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741851_1027 (size=7818)
2024-12-02T18:32:28,914 INFO  [M:0;95451c90bed2:46111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.94 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3ca8f92bdc346d7946d0ddab0d557f5
2024-12-02T18:32:28,918 INFO  [M:0;95451c90bed2:46111 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d3ca8f92bdc346d7946d0ddab0d557f5
2024-12-02T18:32:28,932 INFO  [RS:0;95451c90bed2:42095 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:32:28,932 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:32:28,932 INFO  [RS:0;95451c90bed2:42095 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,42095,1733164296771; zookeeper connection closed.
2024-12-02T18:32:28,932 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42095-0x101975065300001, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:32:28,932 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22e4f64e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22e4f64e
2024-12-02T18:32:28,932 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:32:28,934 DEBUG [M:0;95451c90bed2:46111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/41f8535b0fdc4799bb4ecb5a8ec058e8 is 69, key is 95451c90bed2,42095,1733164296771/rs:state/1733164297238/Put/seqid=0
2024-12-02T18:32:28,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741852_1028 (size=5156)
2024-12-02T18:32:28,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741852_1028 (size=5156)
2024-12-02T18:32:28,944 INFO  [M:0;95451c90bed2:46111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/41f8535b0fdc4799bb4ecb5a8ec058e8
2024-12-02T18:32:28,964 DEBUG [M:0;95451c90bed2:46111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/803e091ddd254afca9ea8c7e6b85f822 is 52, key is load_balancer_on/state:d/1733164298006/Put/seqid=0
2024-12-02T18:32:28,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741853_1029 (size=5056)
2024-12-02T18:32:28,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741853_1029 (size=5056)
2024-12-02T18:32:28,970 INFO  [M:0;95451c90bed2:46111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/803e091ddd254afca9ea8c7e6b85f822
2024-12-02T18:32:28,975 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a95a1d22d4149ebb7e84055c1da234c as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a95a1d22d4149ebb7e84055c1da234c
2024-12-02T18:32:28,981 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a95a1d22d4149ebb7e84055c1da234c, entries=8, sequenceid=121, filesize=5.5 K
2024-12-02T18:32:28,983 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3ca8f92bdc346d7946d0ddab0d557f5 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d3ca8f92bdc346d7946d0ddab0d557f5
2024-12-02T18:32:28,988 INFO  [M:0;95451c90bed2:46111 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d3ca8f92bdc346d7946d0ddab0d557f5
2024-12-02T18:32:28,988 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d3ca8f92bdc346d7946d0ddab0d557f5, entries=14, sequenceid=121, filesize=7.6 K
2024-12-02T18:32:28,989 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/41f8535b0fdc4799bb4ecb5a8ec058e8 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/41f8535b0fdc4799bb4ecb5a8ec058e8
2024-12-02T18:32:28,996 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/41f8535b0fdc4799bb4ecb5a8ec058e8, entries=1, sequenceid=121, filesize=5.0 K
2024-12-02T18:32:28,997 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/803e091ddd254afca9ea8c7e6b85f822 as hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/803e091ddd254afca9ea8c7e6b85f822
2024-12-02T18:32:29,006 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34333/user/jenkins/test-data/d161d5f9-8f06-f747-7d05-8d8860bb02ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/803e091ddd254afca9ea8c7e6b85f822, entries=1, sequenceid=121, filesize=4.9 K
2024-12-02T18:32:29,007 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.54 KB/44590, heapSize ~54.85 KB/56168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=121, compaction requested=false
2024-12-02T18:32:29,008 INFO  [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:29,008 DEBUG [M:0;95451c90bed2:46111 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164348861Disabling compacts and flushes for region at 1733164348861Disabling writes for close at 1733164348861Obtaining lock to block concurrent updates at 1733164348861Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164348861Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44590, getHeapSize=56168, getOffHeapSize=0, getCellsCount=140 at 1733164348861Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164348862 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164348862Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164348876 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164348876Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164348885 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164348901 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164348902 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164348918 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164348933 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164348933Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164348949 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164348964 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164348964Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360058f8: reopening flushed file at 1733164348974 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@519f17af: reopening flushed file at 1733164348982 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d77f13b: reopening flushed file at 1733164348988 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56a45f55: reopening flushed file at 1733164348996 (+8 ms)Finished flush of dataSize ~43.54 KB/44590, heapSize ~54.85 KB/56168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=121, compaction requested=false at 1733164349007 (+11 ms)Writing region close event to WAL at 1733164349008 (+1 ms)Closed at 1733164349008
2024-12-02T18:32:29,008 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:29,008 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:29,009 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:29,009 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:29,009 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:32:29,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40575 is added to blk_1073741830_1006 (size=52987)
2024-12-02T18:32:29,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741830_1006 (size=52987)
2024-12-02T18:32:29,011 INFO  [M:0;95451c90bed2:46111 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:32:29,011 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:32:29,011 INFO  [M:0;95451c90bed2:46111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46111
2024-12-02T18:32:29,011 INFO  [M:0;95451c90bed2:46111 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:32:29,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:32:29,132 INFO  [M:0;95451c90bed2:46111 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:32:29,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46111-0x101975065300000, quorum=127.0.0.1:54662, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:32:29,163 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c734161{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:32:29,164 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75e789f0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:32:29,164 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:32:29,164 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14a6d451{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:32:29,164 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34534d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,STOPPED}
2024-12-02T18:32:29,166 WARN  [BP-2030169084-172.17.0.3-1733164294364 heartbeating to localhost/127.0.0.1:34333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:32:29,166 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:32:29,166 WARN  [BP-2030169084-172.17.0.3-1733164294364 heartbeating to localhost/127.0.0.1:34333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2030169084-172.17.0.3-1733164294364 (Datanode Uuid c9bc10d2-4846-482e-b103-8b87a4a84949) service to localhost/127.0.0.1:34333
2024-12-02T18:32:29,166 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:32:29,167 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data3/current/BP-2030169084-172.17.0.3-1733164294364 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:32:29,167 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data4/current/BP-2030169084-172.17.0.3-1733164294364 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:32:29,167 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:32:29,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40527499{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:32:29,170 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c1ed8d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:32:29,170 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:32:29,170 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c13156e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:32:29,170 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4faae08f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,STOPPED}
2024-12-02T18:32:29,171 WARN  [BP-2030169084-172.17.0.3-1733164294364 heartbeating to localhost/127.0.0.1:34333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:32:29,171 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:32:29,172 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:32:29,172 WARN  [BP-2030169084-172.17.0.3-1733164294364 heartbeating to localhost/127.0.0.1:34333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2030169084-172.17.0.3-1733164294364 (Datanode Uuid 46f9f76c-5e8d-43a0-b072-c9fbf4c45e9a) service to localhost/127.0.0.1:34333
2024-12-02T18:32:29,172 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data1/current/BP-2030169084-172.17.0.3-1733164294364 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:32:29,172 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/cluster_30ebbc26-de3d-26f9-d96e-d76b1717d2fe/data/data2/current/BP-2030169084-172.17.0.3-1733164294364 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:32:29,173 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:32:29,179 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43498b11{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:32:29,179 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:32:29,179 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:32:29,179 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:32:29,180 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir/,STOPPED}
2024-12-02T18:32:29,185 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:32:29,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:32:29,210 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 179)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34333
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34333
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34333 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34333
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34333 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:34333
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34333 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34333
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34333
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=3954 (was 4122)
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=302, ProcessCount=11, AvailableMemoryMB=3954
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.log.dir so I do NOT create it in target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b368016b-a0d6-1035-5806-c097fc5e0626/hadoop.tmp.dir so I do NOT create it in target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b, deleteOnExit=true
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/test.cache.data in system properties and HBase conf
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:32:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:32:29,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:32:29,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:32:29,220 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:32:29,232 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:32:29,263 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:32:29,595 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:32:29,598 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:32:29,609 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:32:29,609 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:32:29,610 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:32:29,610 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:32:29,611 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dab95de{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:32:29,611 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1776bdc2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:32:29,707 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2735da07{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/java.io.tmpdir/jetty-localhost-40505-hadoop-hdfs-3_4_1-tests_jar-_-any-189747156835617070/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:32:29,708 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@425d5d71{HTTP/1.1, (http/1.1)}{localhost:40505}
2024-12-02T18:32:29,708 INFO  [Time-limited test {}] server.Server(415): Started @245094ms
2024-12-02T18:32:29,720 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:32:29,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:29,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:29,990 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:32:29,992 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:32:29,993 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:32:29,993 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:32:29,993 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:32:29,993 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31e425dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:32:29,993 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3523e770{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:32:30,095 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c9ba3de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/java.io.tmpdir/jetty-localhost-35087-hadoop-hdfs-3_4_1-tests_jar-_-any-16474074386936184113/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:32:30,095 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70d55230{HTTP/1.1, (http/1.1)}{localhost:35087}
2024-12-02T18:32:30,095 INFO  [Time-limited test {}] server.Server(415): Started @245481ms
2024-12-02T18:32:30,096 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:32:30,133 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:32:30,135 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:32:30,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:32:30,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:32:30,136 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-02T18:32:30,136 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@216b0c63{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:32:30,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dc4994c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:32:30,255 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2998c4ad{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/java.io.tmpdir/jetty-localhost-42147-hadoop-hdfs-3_4_1-tests_jar-_-any-14987696017094596680/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:32:30,255 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9caeb33{HTTP/1.1, (http/1.1)}{localhost:42147}
2024-12-02T18:32:30,255 INFO  [Time-limited test {}] server.Server(415): Started @245641ms
2024-12-02T18:32:30,257 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:32:30,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:30,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:31,589 WARN  [Thread-1955 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data1/current/BP-1257207442-172.17.0.3-1733164349243/current, will proceed with Du for space computation calculation, 
2024-12-02T18:32:31,589 WARN  [Thread-1956 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data2/current/BP-1257207442-172.17.0.3-1733164349243/current, will proceed with Du for space computation calculation, 
2024-12-02T18:32:31,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:32:31,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:32:31,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:32:31,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-02T18:32:31,609 WARN  [Thread-1919 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:32:31,611 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x879828fda4427fc4 with lease ID 0x599dd458c8c17328: Processing first storage report for DS-0d3ad8bb-6630-4bc9-96e2-0f4e47a09419 from datanode DatanodeRegistration(127.0.0.1:41107, datanodeUuid=c83283a7-65c0-4a22-9875-bad767033ef2, infoPort=39097, infoSecurePort=0, ipcPort=35703, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243)
2024-12-02T18:32:31,611 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x879828fda4427fc4 with lease ID 0x599dd458c8c17328: from storage DS-0d3ad8bb-6630-4bc9-96e2-0f4e47a09419 node DatanodeRegistration(127.0.0.1:41107, datanodeUuid=c83283a7-65c0-4a22-9875-bad767033ef2, infoPort=39097, infoSecurePort=0, ipcPort=35703, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:32:31,611 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x879828fda4427fc4 with lease ID 0x599dd458c8c17328: Processing first storage report for DS-14bc0adc-6eef-49f5-842e-af3408f33855 from datanode DatanodeRegistration(127.0.0.1:41107, datanodeUuid=c83283a7-65c0-4a22-9875-bad767033ef2, infoPort=39097, infoSecurePort=0, ipcPort=35703, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243)
2024-12-02T18:32:31,611 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x879828fda4427fc4 with lease ID 0x599dd458c8c17328: from storage DS-14bc0adc-6eef-49f5-842e-af3408f33855 node DatanodeRegistration(127.0.0.1:41107, datanodeUuid=c83283a7-65c0-4a22-9875-bad767033ef2, infoPort=39097, infoSecurePort=0, ipcPort=35703, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:32:31,734 WARN  [Thread-1967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data4/current/BP-1257207442-172.17.0.3-1733164349243/current, will proceed with Du for space computation calculation, 
2024-12-02T18:32:31,734 WARN  [Thread-1966 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data3/current/BP-1257207442-172.17.0.3-1733164349243/current, will proceed with Du for space computation calculation, 
2024-12-02T18:32:31,750 WARN  [Thread-1942 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:32:31,752 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xee4a18482c042c6a with lease ID 0x599dd458c8c17329: Processing first storage report for DS-59c1da62-121e-4680-ae0c-341eca1eb6d0 from datanode DatanodeRegistration(127.0.0.1:37931, datanodeUuid=73b8d92b-9718-4fca-bb02-de9a72875ec0, infoPort=35877, infoSecurePort=0, ipcPort=44585, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243)
2024-12-02T18:32:31,752 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee4a18482c042c6a with lease ID 0x599dd458c8c17329: from storage DS-59c1da62-121e-4680-ae0c-341eca1eb6d0 node DatanodeRegistration(127.0.0.1:37931, datanodeUuid=73b8d92b-9718-4fca-bb02-de9a72875ec0, infoPort=35877, infoSecurePort=0, ipcPort=44585, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:32:31,752 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xee4a18482c042c6a with lease ID 0x599dd458c8c17329: Processing first storage report for DS-0adad9dc-b19f-4fe3-a75e-7365aa5e4df0 from datanode DatanodeRegistration(127.0.0.1:37931, datanodeUuid=73b8d92b-9718-4fca-bb02-de9a72875ec0, infoPort=35877, infoSecurePort=0, ipcPort=44585, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243)
2024-12-02T18:32:31,752 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee4a18482c042c6a with lease ID 0x599dd458c8c17329: from storage DS-0adad9dc-b19f-4fe3-a75e-7365aa5e4df0 node DatanodeRegistration(127.0.0.1:37931, datanodeUuid=73b8d92b-9718-4fca-bb02-de9a72875ec0, infoPort=35877, infoSecurePort=0, ipcPort=44585, storageInfo=lv=-57;cid=testClusterID;nsid=136112207;c=1733164349243), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:32:31,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:31,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:31,795 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677
2024-12-02T18:32:31,798 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/zookeeper_0, clientPort=51665, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:32:31,800 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51665
2024-12-02T18:32:31,800 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,802 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:32:31,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:32:31,814 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b with version=8
2024-12-02T18:32:31,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:32:31,816 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:32:31,817 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33279
2024-12-02T18:32:31,818 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33279 connecting to ZooKeeper ensemble=127.0.0.1:51665
2024-12-02T18:32:31,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:332790x0, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:32:31,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33279-0x10197513ce10000 connected
2024-12-02T18:32:31,956 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,957 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,959 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:32:31,960 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b, hbase.cluster.distributed=false
2024-12-02T18:32:31,962 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:32:31,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33279
2024-12-02T18:32:31,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33279
2024-12-02T18:32:31,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33279
2024-12-02T18:32:31,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33279
2024-12-02T18:32:31,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33279
2024-12-02T18:32:31,982 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:32:31,983 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43681
2024-12-02T18:32:31,985 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43681 connecting to ZooKeeper ensemble=127.0.0.1:51665
2024-12-02T18:32:31,985 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,987 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:31,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436810x0, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:32:31,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436810x0, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:32:31,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43681-0x10197513ce10001 connected
2024-12-02T18:32:31,998 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:32:31,999 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:32:31,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:32:32,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:32:32,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43681
2024-12-02T18:32:32,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43681
2024-12-02T18:32:32,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43681
2024-12-02T18:32:32,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43681
2024-12-02T18:32:32,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43681
2024-12-02T18:32:32,015 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:33279
2024-12-02T18:32:32,015 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:32:32,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:32:32,019 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:32:32,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,029 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:32:32,030 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,33279,1733164351816 from backup master directory
2024-12-02T18:32:32,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:32:32,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:32:32,040 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:32:32,040 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,044 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/hbase.id] with ID: 225fd3e3-acc8-4824-9b19-5d7b01602c4e
2024-12-02T18:32:32,044 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/.tmp/hbase.id
2024-12-02T18:32:32,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:32:32,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:32:32,051 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/.tmp/hbase.id]:[hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/hbase.id]
2024-12-02T18:32:32,063 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:32,063 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:32:32,064 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-02T18:32:32,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:32:32,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:32:32,078 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:32:32,078 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:32:32,078 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:32:32,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:32:32,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:32:32,085 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store
2024-12-02T18:32:32,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:32:32,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:32:32,092 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:32,092 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:32:32,092 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164352092Disabling compacts and flushes for region at 1733164352092Disabling writes for close at 1733164352092Writing region close event to WAL at 1733164352092Closed at 1733164352092
2024-12-02T18:32:32,093 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/.initializing
2024-12-02T18:32:32,093 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/WALs/95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,095 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C33279%2C1733164351816, suffix=, logDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/WALs/95451c90bed2,33279,1733164351816, archiveDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/oldWALs, maxLogs=10
2024-12-02T18:32:32,096 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C33279%2C1733164351816.1733164352096
2024-12-02T18:32:32,101 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/WALs/95451c90bed2,33279,1733164351816/95451c90bed2%2C33279%2C1733164351816.1733164352096
2024-12-02T18:32:32,104 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35877:35877)]
2024-12-02T18:32:32,105 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:32:32,105 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:32,105 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,105 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,107 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:32:32,109 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:32:32,109 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,109 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,109 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:32:32,110 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:32:32,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:32,111 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:32:32,112 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:32:32,112 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,113 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:32,113 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:32:32,114 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:32:32,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:32,115 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,115 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,116 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,117 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,117 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,117 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:32:32,118 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:32:32,120 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:32:32,121 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826849, jitterRate=0.051392972469329834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:32:32,121 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164352105Initializing all the Stores at 1733164352106 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352106Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164352107 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164352107Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164352107Cleaning up temporary data from old regions at 1733164352117 (+10 ms)Region opened successfully at 1733164352121 (+4 ms)
2024-12-02T18:32:32,124 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:32:32,128 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5d004b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:32:32,128 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:32:32,128 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:32:32,129 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:32:32,129 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:32:32,129 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:32:32,130 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:32:32,130 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:32:32,134 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:32:32,135 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:32:32,145 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:32:32,145 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:32:32,146 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:32:32,155 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:32:32,156 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:32:32,157 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:32:32,166 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:32:32,167 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:32:32,176 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:32:32,179 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:32:32,187 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:32:32,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:32:32,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:32:32,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,198 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,33279,1733164351816, sessionid=0x10197513ce10000, setting cluster-up flag (Was=false)
2024-12-02T18:32:32,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,250 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:32:32,251 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,303 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:32:32,304 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,33279,1733164351816
2024-12-02T18:32:32,305 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:32:32,306 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:32:32,307 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:32:32,307 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:32:32,307 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,33279,1733164351816
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:32:32,308 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164382309
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:32:32,309 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:32:32,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:32:32,310 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:32:32,310 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164352310,5,FailOnTimeoutGroup]
2024-12-02T18:32:32,310 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164352310,5,FailOnTimeoutGroup]
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,310 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,311 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,311 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:32:32,317 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:32:32,317 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:32:32,318 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:32:32,318 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b
2024-12-02T18:32:32,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:32:32,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:32:32,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:32,325 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:32:32,326 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:32:32,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,327 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,327 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:32:32,328 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:32:32,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:32:32,330 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:32:32,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:32:32,331 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:32:32,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,332 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:32:32,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740
2024-12-02T18:32:32,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740
2024-12-02T18:32:32,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:32:32,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:32:32,334 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:32:32,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:32:32,339 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:32:32,339 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771394, jitterRate=-0.019121944904327393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:32:32,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164352324Initializing all the Stores at 1733164352325 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352325Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352325Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164352325Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352325Cleaning up temporary data from old regions at 1733164352334 (+9 ms)Region opened successfully at 1733164352340 (+6 ms)
2024-12-02T18:32:32,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:32:32,340 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:32:32,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:32:32,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:32:32,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:32:32,340 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:32:32,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164352340Disabling compacts and flushes for region at 1733164352340Disabling writes for close at 1733164352340Writing region close event to WAL at 1733164352340Closed at 1733164352340
2024-12-02T18:32:32,342 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:32:32,342 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:32:32,342 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:32:32,343 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:32:32,344 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:32:32,405 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(746): ClusterId : 225fd3e3-acc8-4824-9b19-5d7b01602c4e
2024-12-02T18:32:32,405 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:32:32,409 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:32:32,409 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:32:32,419 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:32:32,420 DEBUG [RS:0;95451c90bed2:43681 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa19893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:32:32,432 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:43681
2024-12-02T18:32:32,432 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:32:32,432 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:32:32,432 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:32:32,433 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,33279,1733164351816 with port=43681, startcode=1733164351982
2024-12-02T18:32:32,433 DEBUG [RS:0;95451c90bed2:43681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:32:32,435 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52293, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:32:32,435 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33279 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,435 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33279 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,437 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b
2024-12-02T18:32:32,437 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44943
2024-12-02T18:32:32,437 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:32:32,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:32:32,450 DEBUG [RS:0;95451c90bed2:43681 {}] zookeeper.ZKUtil(111): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,450 WARN  [RS:0;95451c90bed2:43681 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:32:32,450 INFO  [RS:0;95451c90bed2:43681 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:32:32,451 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,451 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,43681,1733164351982]
2024-12-02T18:32:32,454 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:32:32,455 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:32:32,455 INFO  [RS:0;95451c90bed2:43681 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:32:32,455 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,456 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:32:32,457 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:32:32,457 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:32:32,457 DEBUG [RS:0;95451c90bed2:43681 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:32:32,457 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,457 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,457 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,458 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,458 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,458 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43681,1733164351982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:32:32,488 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:32:32,489 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,43681,1733164351982-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,489 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,489 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.Replication(171): 95451c90bed2,43681,1733164351982 started
2024-12-02T18:32:32,494 WARN  [95451c90bed2:33279 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:32:32,500 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:32,500 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,43681,1733164351982, RpcServer on 95451c90bed2/172.17.0.3:43681, sessionid=0x10197513ce10001
2024-12-02T18:32:32,500 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:32:32,500 DEBUG [RS:0;95451c90bed2:43681 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,500 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,43681,1733164351982'
2024-12-02T18:32:32,500 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:32:32,500 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,43681,1733164351982'
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:32:32,501 DEBUG [RS:0;95451c90bed2:43681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:32:32,501 INFO  [RS:0;95451c90bed2:43681 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:32:32,501 INFO  [RS:0;95451c90bed2:43681 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:32:32,605 INFO  [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C43681%2C1733164351982, suffix=, logDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982, archiveDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs, maxLogs=32
2024-12-02T18:32:32,606 INFO  [RS:0;95451c90bed2:43681 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43681%2C1733164351982.1733164352606
2024-12-02T18:32:32,615 INFO  [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164352606
2024-12-02T18:32:32,616 DEBUG [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35877:35877)]
2024-12-02T18:32:32,745 DEBUG [95451c90bed2:33279 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:32:32,745 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,747 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,43681,1733164351982, state=OPENING
2024-12-02T18:32:32,755 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:32:32,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:32,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:32:32,766 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:32:32,766 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:32:32,766 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:32:32,766 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,43681,1733164351982}]
2024-12-02T18:32:32,920 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:32:32,924 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40027, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:32:32,931 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:32:32,932 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:32:32,934 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C43681%2C1733164351982.meta, suffix=.meta, logDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982, archiveDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs, maxLogs=32
2024-12-02T18:32:32,935 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43681%2C1733164351982.meta.1733164352934.meta
2024-12-02T18:32:32,939 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.meta.1733164352934.meta
2024-12-02T18:32:32,940 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35877:35877)]
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:32:32,941 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:32:32,941 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:32:32,942 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:32:32,943 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:32:32,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:32:32,944 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:32:32,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:32:32,945 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:32:32,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,945 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:32:32,946 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:32:32,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:32,946 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:32:32,946 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:32:32,947 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740
2024-12-02T18:32:32,948 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740
2024-12-02T18:32:32,949 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:32:32,949 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:32:32,950 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:32:32,951 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:32:32,952 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868008, jitterRate=0.1037302166223526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:32:32,952 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:32:32,952 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164352941Writing region info on filesystem at 1733164352941Initializing all the Stores at 1733164352942 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352942Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352942Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164352942Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164352942Cleaning up temporary data from old regions at 1733164352949 (+7 ms)Running coprocessor post-open hooks at 1733164352952 (+3 ms)Region opened successfully at 1733164352952
2024-12-02T18:32:32,953 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164352919
2024-12-02T18:32:32,955 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:32:32,955 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:32:32,956 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,957 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,43681,1733164351982, state=OPEN
2024-12-02T18:32:32,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:32:32,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:32:32,998 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,43681,1733164351982
2024-12-02T18:32:32,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:32:32,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:32:33,001 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:32:33,001 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,43681,1733164351982 in 232 msec
2024-12-02T18:32:33,005 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:32:33,005 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 659 msec
2024-12-02T18:32:33,007 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:32:33,007 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:32:33,008 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:32:33,008 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,43681,1733164351982, seqNum=-1]
2024-12-02T18:32:33,009 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:32:33,010 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:32:33,015 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 708 msec
2024-12-02T18:32:33,015 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164353015, completionTime=-1
2024-12-02T18:32:33,015 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:32:33,015 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164413017
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164473017
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:33279, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,017 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,019 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.981sec
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:32:33,021 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:32:33,022 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:32:33,022 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:32:33,025 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:32:33,025 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:32:33,025 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,33279,1733164351816-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:32:33,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6182ce6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:32:33,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,33279,-1 for getting cluster id
2024-12-02T18:32:33,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:32:33,108 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '225fd3e3-acc8-4824-9b19-5d7b01602c4e'
2024-12-02T18:32:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:32:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "225fd3e3-acc8-4824-9b19-5d7b01602c4e"

2024-12-02T18:32:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1128e26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:32:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,33279,-1]
2024-12-02T18:32:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:32:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:32:33,111 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:32:33,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6d286b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:32:33,113 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:32:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,43681,1733164351982, seqNum=-1]
2024-12-02T18:32:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:32:33,117 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:32:33,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,33279,1733164351816
2024-12-02T18:32:33,120 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:32:33,123 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:32:33,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-02T18:32:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 95451c90bed2,33279,1733164351816
2024-12-02T18:32:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d63bf95
2024-12-02T18:32:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-02T18:32:33,125 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-02T18:32:33,126 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-02T18:32:33,126 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-02T18:32:33,126 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:32:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-02T18:32:33,128 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-02T18:32:33,128 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:33,129 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-02T18:32:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:32:33,130 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-02T18:32:33,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741835_1011 (size=381)
2024-12-02T18:32:33,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741835_1011 (size=381)
2024-12-02T18:32:33,137 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ec829d4c71c771c7e68b3e02000f86a8, NAME => 'TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b
2024-12-02T18:32:33,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741836_1012 (size=64)
2024-12-02T18:32:33,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741836_1012 (size=64)
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing ec829d4c71c771c7e68b3e02000f86a8, disabling compactions & flushes
2024-12-02T18:32:33,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8. after waiting 0 ms
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ec829d4c71c771c7e68b3e02000f86a8:
Waiting for close lock at 1733164353144Disabling compacts and flushes for region at 1733164353144Disabling writes for close at 1733164353144Writing region close event to WAL at 1733164353144Closed at 1733164353144
2024-12-02T18:32:33,146 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-02T18:32:33,146 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733164353146"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164353146"}]},"ts":"1733164353146"}
2024-12-02T18:32:33,148 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-02T18:32:33,150 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-02T18:32:33,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164353150"}]},"ts":"1733164353150"}
2024-12-02T18:32:33,152 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-02T18:32:33,152 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, ASSIGN}]
2024-12-02T18:32:33,154 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, ASSIGN
2024-12-02T18:32:33,155 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, ASSIGN; state=OFFLINE, location=95451c90bed2,43681,1733164351982; forceNewPlan=false, retain=false
2024-12-02T18:32:33,305 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ec829d4c71c771c7e68b3e02000f86a8, regionState=OPENING, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:33,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, ASSIGN because future has completed
2024-12-02T18:32:33,310 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982}]
2024-12-02T18:32:33,469 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,469 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ec829d4c71c771c7e68b3e02000f86a8, NAME => 'TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:32:33,469 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,470 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:33,470 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,470 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,471 INFO  [StoreOpener-ec829d4c71c771c7e68b3e02000f86a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ec829d4c71c771c7e68b3e02000f86a8 
2024-12-02T18:32:33,473 INFO  [StoreOpener-ec829d4c71c771c7e68b3e02000f86a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec829d4c71c771c7e68b3e02000f86a8 columnFamilyName info
2024-12-02T18:32:33,473 DEBUG [StoreOpener-ec829d4c71c771c7e68b3e02000f86a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:33,474 INFO  [StoreOpener-ec829d4c71c771c7e68b3e02000f86a8-1 {}] regionserver.HStore(327): Store=ec829d4c71c771c7e68b3e02000f86a8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:33,474 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,476 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,476 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,477 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,477 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,479 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,481 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:32:33,482 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ec829d4c71c771c7e68b3e02000f86a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728101, jitterRate=-0.0741717666387558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:32:33,482 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:33,482 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ec829d4c71c771c7e68b3e02000f86a8:
Running coprocessor pre-open hook at 1733164353470Writing region info on filesystem at 1733164353470Initializing all the Stores at 1733164353471 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164353471Cleaning up temporary data from old regions at 1733164353477 (+6 ms)Running coprocessor post-open hooks at 1733164353482 (+5 ms)Region opened successfully at 1733164353482
2024-12-02T18:32:33,484 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., pid=6, masterSystemTime=1733164353465
2024-12-02T18:32:33,486 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,486 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:33,487 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ec829d4c71c771c7e68b3e02000f86a8, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:33,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982 because future has completed
2024-12-02T18:32:33,493 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-02T18:32:33,493 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982 in 181 msec
2024-12-02T18:32:33,496 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-02T18:32:33,496 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, ASSIGN in 341 msec
2024-12-02T18:32:33,497 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-02T18:32:33,497 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733164353497"}]},"ts":"1733164353497"}
2024-12-02T18:32:33,499 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-02T18:32:33,501 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-02T18:32:33,501 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,501 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,501 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,501 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,502 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,502 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,502 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,502 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,503 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 375 msec
2024-12-02T18:32:33,516 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,520 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:33,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:33,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:34,026 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:32:34,030 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,030 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,031 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,050 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:34,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:34,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:35,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:35,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:36,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:36,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:37,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:37,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:38,454 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:32:38,455 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-02T18:32:38,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:38,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:38,972 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:32:38,974 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,974 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,975 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,975 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,975 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,976 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,976 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:38,977 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,001 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,002 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,002 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,002 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,002 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,002 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:32:39,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:39,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:40,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:40,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:41,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:32:41,591 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-02T18:32:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:32:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-02T18:32:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:32:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-02T18:32:41,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-02T18:32:41,592 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-02T18:32:41,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:41,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:42,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:42,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-02T18:32:43,186 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-02T18:32:43,186 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-02T18:32:43,189 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-02T18:32:43,189 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2]
2024-12-02T18:32:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:43,209 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec829d4c71c771c7e68b3e02000f86a8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:32:43,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/387fcadaf8c946be93959b91402f8bc3 is 1080, key is row0001/info:/1733164363194/Put/seqid=0
2024-12-02T18:32:43,267 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-02T18:32:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:45060 deadline: 1733164373266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982
2024-12-02T18:32:43,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741837_1013 (size=12509)
2024-12-02T18:32:43,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741837_1013 (size=12509)
2024-12-02T18:32:43,293 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/387fcadaf8c946be93959b91402f8bc3
2024-12-02T18:32:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/387fcadaf8c946be93959b91402f8bc3 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3
2024-12-02T18:32:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-02T18:32:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-02T18:32:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 because the exception is null or not the one we care about
2024-12-02T18:32:43,310 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3, entries=7, sequenceid=11, filesize=12.2 K
2024-12-02T18:32:43,312 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for ec829d4c71c771c7e68b3e02000f86a8 in 102ms, sequenceid=11, compaction requested=false
2024-12-02T18:32:43,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:43,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:43,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:44,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:44,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:45,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:45,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:46,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:46,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:47,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:47,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:48,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:48,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:49,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:49,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:50,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:50,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:51,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:51,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:52,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:52,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:53,367 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec829d4c71c771c7e68b3e02000f86a8 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-02T18:32:53,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/858796a200f040aa95e582dd577063ac is 1080, key is row0008/info:/1733164363210/Put/seqid=0
2024-12-02T18:32:53,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741838_1014 (size=29761)
2024-12-02T18:32:53,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741838_1014 (size=29761)
2024-12-02T18:32:53,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:53,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:53,780 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/858796a200f040aa95e582dd577063ac
2024-12-02T18:32:53,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/858796a200f040aa95e582dd577063ac as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac
2024-12-02T18:32:53,793 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac, entries=23, sequenceid=37, filesize=29.1 K
2024-12-02T18:32:53,795 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for ec829d4c71c771c7e68b3e02000f86a8 in 427ms, sequenceid=37, compaction requested=false
2024-12-02T18:32:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-12-02T18:32:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac because midkey is the same as first or last row
2024-12-02T18:32:54,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:54,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,380 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec829d4c71c771c7e68b3e02000f86a8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:32:55,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/12999f58bfb64db29d1405309fe32053 is 1080, key is row0031/info:/1733164373368/Put/seqid=0
2024-12-02T18:32:55,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741839_1015 (size=12509)
2024-12-02T18:32:55,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741839_1015 (size=12509)
2024-12-02T18:32:55,398 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/12999f58bfb64db29d1405309fe32053
2024-12-02T18:32:55,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/12999f58bfb64db29d1405309fe32053 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053
2024-12-02T18:32:55,411 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053, entries=7, sequenceid=47, filesize=12.2 K
2024-12-02T18:32:55,412 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for ec829d4c71c771c7e68b3e02000f86a8 in 32ms, sequenceid=47, compaction requested=true
2024-12-02T18:32:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac because midkey is the same as first or last row
2024-12-02T18:32:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec829d4c71c771c7e68b3e02000f86a8:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:32:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:55,413 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:32:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,413 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec829d4c71c771c7e68b3e02000f86a8 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-02T18:32:55,414 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:32:55,415 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): ec829d4c71c771c7e68b3e02000f86a8/info is initiating minor compaction (all files)
2024-12-02T18:32:55,415 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec829d4c71c771c7e68b3e02000f86a8/info in TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,415 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp, totalSize=53.5 K
2024-12-02T18:32:55,415 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387fcadaf8c946be93959b91402f8bc3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733164363194
2024-12-02T18:32:55,416 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858796a200f040aa95e582dd577063ac, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733164363210
2024-12-02T18:32:55,416 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12999f58bfb64db29d1405309fe32053, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733164373368
2024-12-02T18:32:55,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/9024d9c346fc45a79ddbe06e20e7f4df is 1080, key is row0038/info:/1733164375381/Put/seqid=0
2024-12-02T18:32:55,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741840_1016 (size=21141)
2024-12-02T18:32:55,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741840_1016 (size=21141)
2024-12-02T18:32:55,429 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/9024d9c346fc45a79ddbe06e20e7f4df
2024-12-02T18:32:55,435 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec829d4c71c771c7e68b3e02000f86a8#info#compaction#59 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:32:55,436 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/ca0fae0f90fb42288554369e5cc24c1f is 1080, key is row0001/info:/1733164363194/Put/seqid=0
2024-12-02T18:32:55,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/9024d9c346fc45a79ddbe06e20e7f4df as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df
2024-12-02T18:32:55,445 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df, entries=15, sequenceid=65, filesize=20.6 K
2024-12-02T18:32:55,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=11.56 KB/11836 for ec829d4c71c771c7e68b3e02000f86a8 in 33ms, sequenceid=65, compaction requested=false
2024-12-02T18:32:55,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:55,446 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.1 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,446 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,446 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac because midkey is the same as first or last row
2024-12-02T18:32:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,447 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec829d4c71c771c7e68b3e02000f86a8 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-02T18:32:55,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741841_1017 (size=44978)
2024-12-02T18:32:55,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741841_1017 (size=44978)
2024-12-02T18:32:55,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/97a721d2c47d4315a6d94a3f8f051848 is 1080, key is row0053/info:/1733164375415/Put/seqid=0
2024-12-02T18:32:55,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741842_1018 (size=17894)
2024-12-02T18:32:55,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741842_1018 (size=17894)
2024-12-02T18:32:55,457 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/97a721d2c47d4315a6d94a3f8f051848
2024-12-02T18:32:55,457 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/ca0fae0f90fb42288554369e5cc24c1f as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f
2024-12-02T18:32:55,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/97a721d2c47d4315a6d94a3f8f051848 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848
2024-12-02T18:32:55,464 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec829d4c71c771c7e68b3e02000f86a8/info of ec829d4c71c771c7e68b3e02000f86a8 into ca0fae0f90fb42288554369e5cc24c1f(size=43.9 K), total size for store is 64.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:55,464 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., storeName=ec829d4c71c771c7e68b3e02000f86a8/info, priority=13, startTime=1733164375412; duration=0sec
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f because midkey is the same as first or last row
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f because midkey is the same as first or last row
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f because midkey is the same as first or last row
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:55,464 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec829d4c71c771c7e68b3e02000f86a8:info
2024-12-02T18:32:55,468 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848, entries=12, sequenceid=80, filesize=17.5 K
2024-12-02T18:32:55,470 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for ec829d4c71c771c7e68b3e02000f86a8 in 23ms, sequenceid=80, compaction requested=true
2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.0 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f because midkey is the same as first or last row
2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec829d4c71c771c7e68b3e02000f86a8:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:32:55,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:55,470 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:32:55,471 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 84013 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:32:55,471 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): ec829d4c71c771c7e68b3e02000f86a8/info is initiating minor compaction (all files)
2024-12-02T18:32:55,471 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec829d4c71c771c7e68b3e02000f86a8/info in TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,471 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp, totalSize=82.0 K
2024-12-02T18:32:55,471 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0fae0f90fb42288554369e5cc24c1f, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733164363194
2024-12-02T18:32:55,472 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9024d9c346fc45a79ddbe06e20e7f4df, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733164375381
2024-12-02T18:32:55,472 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a721d2c47d4315a6d94a3f8f051848, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733164375415
2024-12-02T18:32:55,484 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec829d4c71c771c7e68b3e02000f86a8#info#compaction#61 average throughput is 32.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:32:55,484 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/2244dd98676c470b9d4029f23fac2551 is 1080, key is row0001/info:/1733164363194/Put/seqid=0
2024-12-02T18:32:55,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741843_1019 (size=74301)
2024-12-02T18:32:55,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741843_1019 (size=74301)
2024-12-02T18:32:55,495 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/.tmp/info/2244dd98676c470b9d4029f23fac2551 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551
2024-12-02T18:32:55,502 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec829d4c71c771c7e68b3e02000f86a8/info of ec829d4c71c771c7e68b3e02000f86a8 into 2244dd98676c470b9d4029f23fac2551(size=72.6 K), total size for store is 72.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:32:55,502 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec829d4c71c771c7e68b3e02000f86a8:

2024-12-02T18:32:55,502 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., storeName=ec829d4c71c771c7e68b3e02000f86a8/info, priority=13, startTime=1733164375470; duration=0sec
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-12-02T18:32:55,503 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-02T18:32:55,504 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:55,504 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:55,504 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec829d4c71c771c7e68b3e02000f86a8:info
2024-12-02T18:32:55,505 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33279 {}] assignment.AssignmentManager(1363): Split request from 95451c90bed2,43681,1733164351982, parent={ENCODED => ec829d4c71c771c7e68b3e02000f86a8, NAME => 'TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-02T18:32:55,512 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33279 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=95451c90bed2,43681,1733164351982
2024-12-02T18:32:55,516 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33279 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ec829d4c71c771c7e68b3e02000f86a8, daughterA=d68881a7558f42336fade15d797e26db, daughterB=f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:55,518 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ec829d4c71c771c7e68b3e02000f86a8, daughterA=d68881a7558f42336fade15d797e26db, daughterB=f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:55,518 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ec829d4c71c771c7e68b3e02000f86a8, daughterA=d68881a7558f42336fade15d797e26db, daughterB=f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:55,518 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ec829d4c71c771c7e68b3e02000f86a8, daughterA=d68881a7558f42336fade15d797e26db, daughterB=f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:55,526 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, UNASSIGN}]
2024-12-02T18:32:55,527 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, UNASSIGN
2024-12-02T18:32:55,529 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ec829d4c71c771c7e68b3e02000f86a8, regionState=CLOSING, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:55,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, UNASSIGN because future has completed
2024-12-02T18:32:55,532 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-02T18:32:55,533 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982}]
2024-12-02T18:32:55,690 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,690 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-02T18:32:55,691 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing ec829d4c71c771c7e68b3e02000f86a8, disabling compactions & flushes
2024-12-02T18:32:55,692 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,692 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,692 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8. after waiting 0 ms
2024-12-02T18:32:55,692 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,693 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848] to archive
2024-12-02T18:32:55,694 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:32:55,696 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/387fcadaf8c946be93959b91402f8bc3
2024-12-02T18:32:55,698 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/858796a200f040aa95e582dd577063ac
2024-12-02T18:32:55,699 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/ca0fae0f90fb42288554369e5cc24c1f
2024-12-02T18:32:55,700 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/12999f58bfb64db29d1405309fe32053
2024-12-02T18:32:55,702 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/9024d9c346fc45a79ddbe06e20e7f4df
2024-12-02T18:32:55,703 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/97a721d2c47d4315a6d94a3f8f051848
2024-12-02T18:32:55,709 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=1
2024-12-02T18:32:55,709 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.
2024-12-02T18:32:55,709 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for ec829d4c71c771c7e68b3e02000f86a8:
Waiting for close lock at 1733164375691Running coprocessor pre-close hooks at 1733164375691Disabling compacts and flushes for region at 1733164375691Disabling writes for close at 1733164375692 (+1 ms)Writing region close event to WAL at 1733164375705 (+13 ms)Running coprocessor post-close hooks at 1733164375709 (+4 ms)Closed at 1733164375709
2024-12-02T18:32:55,712 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,713 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ec829d4c71c771c7e68b3e02000f86a8, regionState=CLOSED
2024-12-02T18:32:55,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982 because future has completed
2024-12-02T18:32:55,717 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-02T18:32:55,717 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure ec829d4c71c771c7e68b3e02000f86a8, server=95451c90bed2,43681,1733164351982 in 182 msec
2024-12-02T18:32:55,719 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-02T18:32:55,719 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ec829d4c71c771c7e68b3e02000f86a8, UNASSIGN in 191 msec
2024-12-02T18:32:55,726 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:55,729 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 1 storefiles, region=ec829d4c71c771c7e68b3e02000f86a8, threads=1
2024-12-02T18:32:55,731 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551 for region: ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741844_1020 (size=27)
2024-12-02T18:32:55,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741844_1020 (size=27)
2024-12-02T18:32:55,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741845_1021 (size=27)
2024-12-02T18:32:55,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741845_1021 (size=27)
2024-12-02T18:32:55,755 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551 for region: ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:32:55,758 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region ec829d4c71c771c7e68b3e02000f86a8 Daughter A: [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8] storefiles, Daughter B: [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8] storefiles.
2024-12-02T18:32:55,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741846_1022 (size=71)
2024-12-02T18:32:55,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741846_1022 (size=71)
2024-12-02T18:32:55,769 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:55,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:55,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:55,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741847_1023 (size=71)
2024-12-02T18:32:55,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741847_1023 (size=71)
2024-12-02T18:32:55,786 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:55,797 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-12-02T18:32:55,801 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-12-02T18:32:55,803 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733164375803"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733164375803"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733164375803"}]},"ts":"1733164375803"}
2024-12-02T18:32:55,804 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733164375803"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164375803"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733164375803"}]},"ts":"1733164375803"}
2024-12-02T18:32:55,804 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733164375803"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733164375803"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733164375803"}]},"ts":"1733164375803"}
2024-12-02T18:32:55,826 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d68881a7558f42336fade15d797e26db, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f517d8dad759fdcbfc38dbdc1488f610, ASSIGN}]
2024-12-02T18:32:55,828 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d68881a7558f42336fade15d797e26db, ASSIGN
2024-12-02T18:32:55,828 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f517d8dad759fdcbfc38dbdc1488f610, ASSIGN
2024-12-02T18:32:55,829 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f517d8dad759fdcbfc38dbdc1488f610, ASSIGN; state=SPLITTING_NEW, location=95451c90bed2,43681,1733164351982; forceNewPlan=false, retain=false
2024-12-02T18:32:55,829 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d68881a7558f42336fade15d797e26db, ASSIGN; state=SPLITTING_NEW, location=95451c90bed2,43681,1733164351982; forceNewPlan=false, retain=false
2024-12-02T18:32:55,980 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f517d8dad759fdcbfc38dbdc1488f610, regionState=OPENING, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:55,980 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d68881a7558f42336fade15d797e26db, regionState=OPENING, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:55,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f517d8dad759fdcbfc38dbdc1488f610, ASSIGN because future has completed
2024-12-02T18:32:55,983 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f517d8dad759fdcbfc38dbdc1488f610, server=95451c90bed2,43681,1733164351982}]
2024-12-02T18:32:55,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d68881a7558f42336fade15d797e26db, ASSIGN because future has completed
2024-12-02T18:32:55,985 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d68881a7558f42336fade15d797e26db, server=95451c90bed2,43681,1733164351982}]
2024-12-02T18:32:56,141 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:32:56,141 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => d68881a7558f42336fade15d797e26db, NAME => 'TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-02T18:32:56,141 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,142 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:56,142 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,142 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,143 INFO  [StoreOpener-d68881a7558f42336fade15d797e26db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d68881a7558f42336fade15d797e26db 
2024-12-02T18:32:56,143 INFO  [StoreOpener-d68881a7558f42336fade15d797e26db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d68881a7558f42336fade15d797e26db columnFamilyName info
2024-12-02T18:32:56,143 DEBUG [StoreOpener-d68881a7558f42336fade15d797e26db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:56,152 DEBUG [StoreOpener-d68881a7558f42336fade15d797e26db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-bottom
2024-12-02T18:32:56,153 INFO  [StoreOpener-d68881a7558f42336fade15d797e26db-1 {}] regionserver.HStore(327): Store=d68881a7558f42336fade15d797e26db/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:56,153 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,154 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,155 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,156 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,156 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,157 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,158 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened d68881a7558f42336fade15d797e26db; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715323, jitterRate=-0.09042005240917206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:32:56,158 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d68881a7558f42336fade15d797e26db
2024-12-02T18:32:56,159 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for d68881a7558f42336fade15d797e26db:
Running coprocessor pre-open hook at 1733164376142Writing region info on filesystem at 1733164376142Initializing all the Stores at 1733164376142Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164376142Cleaning up temporary data from old regions at 1733164376156 (+14 ms)Running coprocessor post-open hooks at 1733164376158 (+2 ms)Region opened successfully at 1733164376159 (+1 ms)
2024-12-02T18:32:56,160 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db., pid=13, masterSystemTime=1733164376136
2024-12-02T18:32:56,160 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store d68881a7558f42336fade15d797e26db:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:32:56,160 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:56,160 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-02T18:32:56,160 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:32:56,160 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1541): d68881a7558f42336fade15d797e26db/info is initiating minor compaction (all files)
2024-12-02T18:32:56,161 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d68881a7558f42336fade15d797e26db/info in TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:32:56,161 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-bottom] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/.tmp, totalSize=72.6 K
2024-12-02T18:32:56,162 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.Compactor(225): Compacting 2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733164363194
2024-12-02T18:32:56,162 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:32:56,162 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:32:56,162 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:32:56,163 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f517d8dad759fdcbfc38dbdc1488f610, NAME => 'TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-02T18:32:56,163 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,163 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:32:56,163 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d68881a7558f42336fade15d797e26db, regionState=OPEN, openSeqNum=86, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:56,163 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,163 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,164 INFO  [StoreOpener-f517d8dad759fdcbfc38dbdc1488f610-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f517d8dad759fdcbfc38dbdc1488f610 
2024-12-02T18:32:56,165 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-02T18:32:56,165 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-02T18:32:56,165 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-12-02T18:32:56,165 INFO  [StoreOpener-f517d8dad759fdcbfc38dbdc1488f610-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f517d8dad759fdcbfc38dbdc1488f610 columnFamilyName info
2024-12-02T18:32:56,165 DEBUG [StoreOpener-f517d8dad759fdcbfc38dbdc1488f610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:32:56,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d68881a7558f42336fade15d797e26db, server=95451c90bed2,43681,1733164351982 because future has completed
2024-12-02T18:32:56,170 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-12-02T18:32:56,170 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d68881a7558f42336fade15d797e26db, server=95451c90bed2,43681,1733164351982 in 182 msec
2024-12-02T18:32:56,172 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d68881a7558f42336fade15d797e26db, ASSIGN in 344 msec
2024-12-02T18:32:56,182 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d68881a7558f42336fade15d797e26db#info#compaction#62 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:32:56,182 DEBUG [StoreOpener-f517d8dad759fdcbfc38dbdc1488f610-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-top
2024-12-02T18:32:56,182 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/.tmp/info/758bb05cd10e42ec84ad97e5a23f986d is 1080, key is row0001/info:/1733164363194/Put/seqid=0
2024-12-02T18:32:56,182 INFO  [StoreOpener-f517d8dad759fdcbfc38dbdc1488f610-1 {}] regionserver.HStore(327): Store=f517d8dad759fdcbfc38dbdc1488f610/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:32:56,182 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,183 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/b71d5770f87b429199b2c4d31e581913 is 193, key is TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610./info:regioninfo/1733164375980/Put/seqid=0
2024-12-02T18:32:56,184 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,185 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,185 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,187 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,187 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f517d8dad759fdcbfc38dbdc1488f610; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844136, jitterRate=0.07337534427642822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-02T18:32:56,188 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:32:56,188 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f517d8dad759fdcbfc38dbdc1488f610:
Running coprocessor pre-open hook at 1733164376163Writing region info on filesystem at 1733164376163Initializing all the Stores at 1733164376164 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164376164Cleaning up temporary data from old regions at 1733164376185 (+21 ms)Running coprocessor post-open hooks at 1733164376188 (+3 ms)Region opened successfully at 1733164376188
2024-12-02T18:32:56,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741848_1024 (size=70862)
2024-12-02T18:32:56,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741848_1024 (size=70862)
2024-12-02T18:32:56,189 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., pid=12, masterSystemTime=1733164376136
2024-12-02T18:32:56,189 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 2
2024-12-02T18:32:56,189 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:56,189 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-02T18:32:56,192 DEBUG [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:32:56,192 INFO  [RS_OPEN_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:32:56,192 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:32:56,192 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:32:56,192 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:32:56,193 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-top] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=72.6 K
2024-12-02T18:32:56,193 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f517d8dad759fdcbfc38dbdc1488f610, regionState=OPEN, openSeqNum=86, regionLocation=95451c90bed2,43681,1733164351982
2024-12-02T18:32:56,193 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733164363194
2024-12-02T18:32:56,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f517d8dad759fdcbfc38dbdc1488f610, server=95451c90bed2,43681,1733164351982 because future has completed
2024-12-02T18:32:56,198 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/.tmp/info/758bb05cd10e42ec84ad97e5a23f986d as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/758bb05cd10e42ec84ad97e5a23f986d
2024-12-02T18:32:56,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741849_1025 (size=9847)
2024-12-02T18:32:56,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741849_1025 (size=9847)
2024-12-02T18:32:56,204 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/b71d5770f87b429199b2c4d31e581913
2024-12-02T18:32:56,206 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-02T18:32:56,206 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in d68881a7558f42336fade15d797e26db/info of d68881a7558f42336fade15d797e26db into 758bb05cd10e42ec84ad97e5a23f986d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:32:56,206 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f517d8dad759fdcbfc38dbdc1488f610, server=95451c90bed2,43681,1733164351982 in 219 msec
2024-12-02T18:32:56,206 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d68881a7558f42336fade15d797e26db:

2024-12-02T18:32:56,206 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db., storeName=d68881a7558f42336fade15d797e26db/info, priority=15, startTime=1733164376160; duration=0sec
2024-12-02T18:32:56,206 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:56,206 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d68881a7558f42336fade15d797e26db:info
2024-12-02T18:32:56,208 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-12-02T18:32:56,208 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f517d8dad759fdcbfc38dbdc1488f610, ASSIGN in 380 msec
2024-12-02T18:32:56,210 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ec829d4c71c771c7e68b3e02000f86a8, daughterA=d68881a7558f42336fade15d797e26db, daughterB=f517d8dad759fdcbfc38dbdc1488f610 in 696 msec
2024-12-02T18:32:56,218 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#64 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:32:56,219 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b6f98f0ab864465487f2d0b6bcff04cf is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:32:56,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741850_1026 (size=8260)
2024-12-02T18:32:56,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741850_1026 (size=8260)
2024-12-02T18:32:56,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/ns/00d2771500904b068b6a4b8efabe9a4d is 43, key is default/ns:d/1733164353011/Put/seqid=0
2024-12-02T18:32:56,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741851_1027 (size=5153)
2024-12-02T18:32:56,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741851_1027 (size=5153)
2024-12-02T18:32:56,230 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/ns/00d2771500904b068b6a4b8efabe9a4d
2024-12-02T18:32:56,230 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b6f98f0ab864465487f2d0b6bcff04cf as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b6f98f0ab864465487f2d0b6bcff04cf
2024-12-02T18:32:56,237 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into b6f98f0ab864465487f2d0b6bcff04cf(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:32:56,237 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:32:56,237 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=15, startTime=1733164376189; duration=0sec
2024-12-02T18:32:56,238 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:32:56,238 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:32:56,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/table/53e8f3a635104da081d17d98197372b2 is 65, key is TestLogRolling-testLogRolling/table:state/1733164353497/Put/seqid=0
2024-12-02T18:32:56,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741852_1028 (size=5340)
2024-12-02T18:32:56,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741852_1028 (size=5340)
2024-12-02T18:32:56,269 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/table/53e8f3a635104da081d17d98197372b2
2024-12-02T18:32:56,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/b71d5770f87b429199b2c4d31e581913 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/info/b71d5770f87b429199b2c4d31e581913
2024-12-02T18:32:56,280 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/info/b71d5770f87b429199b2c4d31e581913, entries=30, sequenceid=17, filesize=9.6 K
2024-12-02T18:32:56,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/ns/00d2771500904b068b6a4b8efabe9a4d as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/ns/00d2771500904b068b6a4b8efabe9a4d
2024-12-02T18:32:56,286 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/ns/00d2771500904b068b6a4b8efabe9a4d, entries=2, sequenceid=17, filesize=5.0 K
2024-12-02T18:32:56,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/table/53e8f3a635104da081d17d98197372b2 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/table/53e8f3a635104da081d17d98197372b2
2024-12-02T18:32:56,291 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/table/53e8f3a635104da081d17d98197372b2, entries=2, sequenceid=17, filesize=5.2 K
2024-12-02T18:32:56,292 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 127ms, sequenceid=17, compaction requested=false
2024-12-02T18:32:56,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-02T18:32:56,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:56,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:45060 deadline: 1733164387448, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8. is not online on 95451c90bed2,43681,1733164351982
2024-12-02T18:32:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8. is not online on 95451c90bed2,43681,1733164351982
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-02T18:32:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8. is not online on 95451c90bed2,43681,1733164351982
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-02T18:32:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733164353126.ec829d4c71c771c7e68b3e02000f86a8., hostname=95451c90bed2,43681,1733164351982, seqNum=2 from cache
2024-12-02T18:32:57,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:57,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:58,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:58,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:59,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:32:59,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:00,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,716 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,733 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,733 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,733 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:00,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:00,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:01,245 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-02T18:33:01,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,249 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,249 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,252 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,252 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,269 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,270 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,270 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,272 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:01,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:01,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:01,795 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-02T18:33:02,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:02,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:03,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:03,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:04,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:04,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:05,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:05,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:06,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:06,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., hostname=95451c90bed2,43681,1733164351982, seqNum=86]
2024-12-02T18:33:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:07,510 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:33:07,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cb671e08fcc04291b695c9a0c6830ef7 is 1080, key is row0065/info:/1733164387499/Put/seqid=0
2024-12-02T18:33:07,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741853_1029 (size=12509)
2024-12-02T18:33:07,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741853_1029 (size=12509)
2024-12-02T18:33:07,519 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cb671e08fcc04291b695c9a0c6830ef7
2024-12-02T18:33:07,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cb671e08fcc04291b695c9a0c6830ef7 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7
2024-12-02T18:33:07,529 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7, entries=7, sequenceid=96, filesize=12.2 K
2024-12-02T18:33:07,530 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for f517d8dad759fdcbfc38dbdc1488f610 in 20ms, sequenceid=96, compaction requested=false
2024-12-02T18:33:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:07,532 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-02T18:33:07,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/57d3f34d4d3349f894b8ec2c35e22096 is 1080, key is row0072/info:/1733164387511/Put/seqid=0
2024-12-02T18:33:07,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741854_1030 (size=16817)
2024-12-02T18:33:07,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741854_1030 (size=16817)
2024-12-02T18:33:07,541 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/57d3f34d4d3349f894b8ec2c35e22096
2024-12-02T18:33:07,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/57d3f34d4d3349f894b8ec2c35e22096 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096
2024-12-02T18:33:07,553 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096, entries=11, sequenceid=110, filesize=16.4 K
2024-12-02T18:33:07,554 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=110, compaction requested=true
2024-12-02T18:33:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:07,554 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:07,554 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-02T18:33:07,555 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37586 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:07,555 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:07,555 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:07,555 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b6f98f0ab864465487f2d0b6bcff04cf, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=36.7 K
2024-12-02T18:33:07,556 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f98f0ab864465487f2d0b6bcff04cf, keycount=3, bloomtype=ROW, size=8.1 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733164375441
2024-12-02T18:33:07,556 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb671e08fcc04291b695c9a0c6830ef7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733164387499
2024-12-02T18:33:07,556 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57d3f34d4d3349f894b8ec2c35e22096, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733164387511
2024-12-02T18:33:07,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b16c4bbdee314cbd9fd59f74307e08da is 1080, key is row0083/info:/1733164387533/Put/seqid=0
2024-12-02T18:33:07,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741855_1031 (size=17894)
2024-12-02T18:33:07,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741855_1031 (size=17894)
2024-12-02T18:33:07,563 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b16c4bbdee314cbd9fd59f74307e08da
2024-12-02T18:33:07,567 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#70 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:07,567 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/670fb0397946450782e880b1dc55bf1c is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:07,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b16c4bbdee314cbd9fd59f74307e08da as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da
2024-12-02T18:33:07,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741856_1032 (size=27778)
2024-12-02T18:33:07,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741856_1032 (size=27778)
2024-12-02T18:33:07,574 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da, entries=12, sequenceid=125, filesize=17.5 K
2024-12-02T18:33:07,575 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=2.10 KB/2152 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=125, compaction requested=false
2024-12-02T18:33:07,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:07,576 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/670fb0397946450782e880b1dc55bf1c as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/670fb0397946450782e880b1dc55bf1c
2024-12-02T18:33:07,582 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 670fb0397946450782e880b1dc55bf1c(size=27.1 K), total size for store is 44.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:07,582 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:07,582 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164387554; duration=0sec
2024-12-02T18:33:07,582 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:07,582 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:07,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:07,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:08,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:08,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:09,571 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:33:09,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/6318078d5c1443b6a823e74513d859a9 is 1080, key is row0095/info:/1733164387555/Put/seqid=0
2024-12-02T18:33:09,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741857_1033 (size=12515)
2024-12-02T18:33:09,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741857_1033 (size=12515)
2024-12-02T18:33:09,584 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/6318078d5c1443b6a823e74513d859a9
2024-12-02T18:33:09,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/6318078d5c1443b6a823e74513d859a9 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9
2024-12-02T18:33:09,594 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9, entries=7, sequenceid=136, filesize=12.2 K
2024-12-02T18:33:09,595 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for f517d8dad759fdcbfc38dbdc1488f610 in 24ms, sequenceid=136, compaction requested=true
2024-12-02T18:33:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:09,596 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:09,596 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-02T18:33:09,597 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 58187 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:09,597 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:09,597 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:09,597 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/670fb0397946450782e880b1dc55bf1c, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=56.8 K
2024-12-02T18:33:09,598 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670fb0397946450782e880b1dc55bf1c, keycount=21, bloomtype=ROW, size=27.1 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733164375441
2024-12-02T18:33:09,598 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16c4bbdee314cbd9fd59f74307e08da, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733164387533
2024-12-02T18:33:09,599 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6318078d5c1443b6a823e74513d859a9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733164387555
2024-12-02T18:33:09,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/df013b38a5e44a3abaea5aa24a15b7c6 is 1080, key is row0102/info:/1733164389573/Put/seqid=0
2024-12-02T18:33:09,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741858_1034 (size=16828)
2024-12-02T18:33:09,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741858_1034 (size=16828)
2024-12-02T18:33:09,606 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/df013b38a5e44a3abaea5aa24a15b7c6
2024-12-02T18:33:09,610 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#73 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:09,611 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/62b3da581804430b9d2626e7f0182ecf is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:09,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/df013b38a5e44a3abaea5aa24a15b7c6 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6
2024-12-02T18:33:09,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741859_1035 (size=48385)
2024-12-02T18:33:09,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741859_1035 (size=48385)
2024-12-02T18:33:09,618 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6, entries=11, sequenceid=150, filesize=16.4 K
2024-12-02T18:33:09,619 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for f517d8dad759fdcbfc38dbdc1488f610 in 23ms, sequenceid=150, compaction requested=false
2024-12-02T18:33:09,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:09,621 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-02T18:33:09,624 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/62b3da581804430b9d2626e7f0182ecf as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/62b3da581804430b9d2626e7f0182ecf
2024-12-02T18:33:09,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/c4ff2ad519484eeaa03e011a35d75fcf is 1080, key is row0113/info:/1733164389598/Put/seqid=0
2024-12-02T18:33:09,631 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 62b3da581804430b9d2626e7f0182ecf(size=47.3 K), total size for store is 63.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:09,631 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:09,631 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164389595; duration=0sec
2024-12-02T18:33:09,631 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:09,631 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:09,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741860_1036 (size=15750)
2024-12-02T18:33:09,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741860_1036 (size=15750)
2024-12-02T18:33:09,632 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/c4ff2ad519484eeaa03e011a35d75fcf
2024-12-02T18:33:09,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/c4ff2ad519484eeaa03e011a35d75fcf as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf
2024-12-02T18:33:09,642 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf, entries=10, sequenceid=163, filesize=15.4 K
2024-12-02T18:33:09,643 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=6.30 KB/6456 for f517d8dad759fdcbfc38dbdc1488f610 in 22ms, sequenceid=163, compaction requested=true
2024-12-02T18:33:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:09,643 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:09,644 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 80963 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:09,644 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:09,644 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:09,645 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/62b3da581804430b9d2626e7f0182ecf, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=79.1 K
2024-12-02T18:33:09,645 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b3da581804430b9d2626e7f0182ecf, keycount=40, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733164375441
2024-12-02T18:33:09,645 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting df013b38a5e44a3abaea5aa24a15b7c6, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733164389573
2024-12-02T18:33:09,646 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ff2ad519484eeaa03e011a35d75fcf, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733164389598
2024-12-02T18:33:09,657 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#75 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:09,657 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/41e181c702124c57b4a856b48d4db0d4 is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:09,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741861_1037 (size=71197)
2024-12-02T18:33:09,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741861_1037 (size=71197)
2024-12-02T18:33:09,666 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/41e181c702124c57b4a856b48d4db0d4 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41e181c702124c57b4a856b48d4db0d4
2024-12-02T18:33:09,671 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 41e181c702124c57b4a856b48d4db0d4(size=69.5 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:09,671 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:09,671 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164389643; duration=0sec
2024-12-02T18:33:09,671 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:09,671 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:09,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:09,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:10,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:10,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:11,640 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:33:11,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/21eb26b9991c4e7a823672fee6359bc8 is 1080, key is row0123/info:/1733164389622/Put/seqid=0
2024-12-02T18:33:11,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741862_1038 (size=12516)
2024-12-02T18:33:11,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741862_1038 (size=12516)
2024-12-02T18:33:11,654 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/21eb26b9991c4e7a823672fee6359bc8
2024-12-02T18:33:11,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/21eb26b9991c4e7a823672fee6359bc8 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8
2024-12-02T18:33:11,665 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8, entries=7, sequenceid=175, filesize=12.2 K
2024-12-02T18:33:11,666 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for f517d8dad759fdcbfc38dbdc1488f610 in 27ms, sequenceid=175, compaction requested=false
2024-12-02T18:33:11,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:11,668 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-02T18:33:11,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/221f12dca58043ae8b1f1e9fe6862678 is 1080, key is row0130/info:/1733164391641/Put/seqid=0
2024-12-02T18:33:11,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741863_1039 (size=17906)
2024-12-02T18:33:11,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741863_1039 (size=17906)
2024-12-02T18:33:11,676 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/221f12dca58043ae8b1f1e9fe6862678
2024-12-02T18:33:11,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/221f12dca58043ae8b1f1e9fe6862678 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678
2024-12-02T18:33:11,686 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678, entries=12, sequenceid=190, filesize=17.5 K
2024-12-02T18:33:11,687 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for f517d8dad759fdcbfc38dbdc1488f610 in 19ms, sequenceid=190, compaction requested=true
2024-12-02T18:33:11,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:11,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:11,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:11,687 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:11,689 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 101619 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:11,689 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:11,689 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:11,689 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-02T18:33:11,689 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41e181c702124c57b4a856b48d4db0d4, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=99.2 K
2024-12-02T18:33:11,689 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e181c702124c57b4a856b48d4db0d4, keycount=61, bloomtype=ROW, size=69.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733164375441
2024-12-02T18:33:11,690 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21eb26b9991c4e7a823672fee6359bc8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733164389622
2024-12-02T18:33:11,690 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221f12dca58043ae8b1f1e9fe6862678, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733164391641
2024-12-02T18:33:11,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f75624dd53324b7dab8ae4c77cf4e310 is 1080, key is row0142/info:/1733164391669/Put/seqid=0
2024-12-02T18:33:11,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741864_1040 (size=15750)
2024-12-02T18:33:11,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741864_1040 (size=15750)
2024-12-02T18:33:11,698 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f75624dd53324b7dab8ae4c77cf4e310
2024-12-02T18:33:11,701 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#79 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:11,702 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/41252cd8c99d4eea9912b26b00eff291 is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:11,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f75624dd53324b7dab8ae4c77cf4e310 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310
2024-12-02T18:33:11,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741865_1041 (size=91854)
2024-12-02T18:33:11,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741865_1041 (size=91854)
2024-12-02T18:33:11,709 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310, entries=10, sequenceid=203, filesize=15.4 K
2024-12-02T18:33:11,710 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/41252cd8c99d4eea9912b26b00eff291 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41252cd8c99d4eea9912b26b00eff291
2024-12-02T18:33:11,710 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=9.46 KB/9684 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=203, compaction requested=false
2024-12-02T18:33:11,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:11,715 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 41252cd8c99d4eea9912b26b00eff291(size=89.7 K), total size for store is 105.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:11,715 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:11,715 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164391687; duration=0sec
2024-12-02T18:33:11,715 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:11,715 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:11,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:11,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:12,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:12,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:13,477 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-02T18:33:13,477 INFO  [master/95451c90bed2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-02T18:33:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:13,715 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-02T18:33:13,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1f41f39dbc094a0286d876994c3c1cc5 is 1080, key is row0152/info:/1733164391690/Put/seqid=0
2024-12-02T18:33:13,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741866_1042 (size=15750)
2024-12-02T18:33:13,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741866_1042 (size=15750)
2024-12-02T18:33:13,729 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1f41f39dbc094a0286d876994c3c1cc5
2024-12-02T18:33:13,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1f41f39dbc094a0286d876994c3c1cc5 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5
2024-12-02T18:33:13,740 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5, entries=10, sequenceid=217, filesize=15.4 K
2024-12-02T18:33:13,741 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=11.56 KB/11836 for f517d8dad759fdcbfc38dbdc1488f610 in 26ms, sequenceid=217, compaction requested=true
2024-12-02T18:33:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:13,741 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:13,742 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123354 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:13,742 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-02T18:33:13,742 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:13,742 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:13,743 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41252cd8c99d4eea9912b26b00eff291, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=120.5 K
2024-12-02T18:33:13,743 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.Compactor(225): Compacting 41252cd8c99d4eea9912b26b00eff291, keycount=80, bloomtype=ROW, size=89.7 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733164375441
2024-12-02T18:33:13,744 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.Compactor(225): Compacting f75624dd53324b7dab8ae4c77cf4e310, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733164391669
2024-12-02T18:33:13,744 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f41f39dbc094a0286d876994c3c1cc5, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733164391690
2024-12-02T18:33:13,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b414505b4df046e6bf08c0340b9c36df is 1080, key is row0162/info:/1733164393718/Put/seqid=0
2024-12-02T18:33:13,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741867_1043 (size=17906)
2024-12-02T18:33:13,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741867_1043 (size=17906)
2024-12-02T18:33:13,751 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b414505b4df046e6bf08c0340b9c36df
2024-12-02T18:33:13,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b414505b4df046e6bf08c0340b9c36df as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df
2024-12-02T18:33:13,759 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#82 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:13,759 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/a42c73160f7f4746bb43f0653336a42c is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:13,763 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df, entries=12, sequenceid=232, filesize=17.5 K
2024-12-02T18:33:13,764 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for f517d8dad759fdcbfc38dbdc1488f610 in 22ms, sequenceid=232, compaction requested=false
2024-12-02T18:33:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:13,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741868_1044 (size=113520)
2024-12-02T18:33:13,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741868_1044 (size=113520)
2024-12-02T18:33:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:13,766 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-02T18:33:13,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/32d41b6a944242d08f10751decb5f076 is 1080, key is row0174/info:/1733164393743/Put/seqid=0
2024-12-02T18:33:13,771 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/a42c73160f7f4746bb43f0653336a42c as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a42c73160f7f4746bb43f0653336a42c
2024-12-02T18:33:13,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741869_1045 (size=16828)
2024-12-02T18:33:13,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741869_1045 (size=16828)
2024-12-02T18:33:13,776 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/32d41b6a944242d08f10751decb5f076
2024-12-02T18:33:13,778 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into a42c73160f7f4746bb43f0653336a42c(size=110.9 K), total size for store is 128.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:13,778 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:13,778 INFO  [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164393741; duration=0sec
2024-12-02T18:33:13,778 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:13,778 DEBUG [RS:0;95451c90bed2:43681-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:13,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/32d41b6a944242d08f10751decb5f076 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076
2024-12-02T18:33:13,786 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076, entries=11, sequenceid=246, filesize=16.4 K
2024-12-02T18:33:13,787 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=246, compaction requested=true
2024-12-02T18:33:13,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:13,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:13,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:13,787 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:13,788 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 148254 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:13,788 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:13,788 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:13,788 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a42c73160f7f4746bb43f0653336a42c, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=144.8 K
2024-12-02T18:33:13,789 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting a42c73160f7f4746bb43f0653336a42c, keycount=100, bloomtype=ROW, size=110.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733164375441
2024-12-02T18:33:13,789 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting b414505b4df046e6bf08c0340b9c36df, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733164393718
2024-12-02T18:33:13,789 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d41b6a944242d08f10751decb5f076, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733164393743
2024-12-02T18:33:13,799 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#84 average throughput is 63.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:13,799 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/551040d256c44dfd9591b0179a8cdb52 is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:13,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741870_1046 (size=138621)
2024-12-02T18:33:13,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741870_1046 (size=138621)
2024-12-02T18:33:13,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:13,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:13,808 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/551040d256c44dfd9591b0179a8cdb52 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/551040d256c44dfd9591b0179a8cdb52
2024-12-02T18:33:13,813 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 551040d256c44dfd9591b0179a8cdb52(size=135.4 K), total size for store is 135.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:13,813 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:13,813 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164393787; duration=0sec
2024-12-02T18:33:13,813 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:13,813 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:14,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:14,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:15,787 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-02T18:33:15,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f53198a2a6f04578b17906307eba0cdd is 1080, key is row0185/info:/1733164393767/Put/seqid=0
2024-12-02T18:33:15,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741871_1047 (size=14674)
2024-12-02T18:33:15,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741871_1047 (size=14674)
2024-12-02T18:33:15,800 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f53198a2a6f04578b17906307eba0cdd
2024-12-02T18:33:15,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/f53198a2a6f04578b17906307eba0cdd as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd
2024-12-02T18:33:15,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:15,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:15,811 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd, entries=9, sequenceid=260, filesize=14.3 K
2024-12-02T18:33:15,812 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=12.61 KB/12912 for f517d8dad759fdcbfc38dbdc1488f610 in 26ms, sequenceid=260, compaction requested=false
2024-12-02T18:33:15,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:15,813 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-02T18:33:15,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/a2067cb3dcd24bfc9d6da415b6b730c5 is 1080, key is row0194/info:/1733164395788/Put/seqid=0
2024-12-02T18:33:15,822 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741872_1048 (size=19013)
2024-12-02T18:33:15,822 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741872_1048 (size=19013)
2024-12-02T18:33:15,823 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/a2067cb3dcd24bfc9d6da415b6b730c5
2024-12-02T18:33:15,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/a2067cb3dcd24bfc9d6da415b6b730c5 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5
2024-12-02T18:33:15,835 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5, entries=13, sequenceid=276, filesize=18.6 K
2024-12-02T18:33:15,836 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for f517d8dad759fdcbfc38dbdc1488f610 in 23ms, sequenceid=276, compaction requested=true
2024-12-02T18:33:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:15,836 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:15,837 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 172308 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:15,838 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:15,838 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:15,838 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-02T18:33:15,838 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/551040d256c44dfd9591b0179a8cdb52, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=168.3 K
2024-12-02T18:33:15,838 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551040d256c44dfd9591b0179a8cdb52, keycount=123, bloomtype=ROW, size=135.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733164375441
2024-12-02T18:33:15,839 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting f53198a2a6f04578b17906307eba0cdd, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733164393767
2024-12-02T18:33:15,839 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2067cb3dcd24bfc9d6da415b6b730c5, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733164395788
2024-12-02T18:33:15,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cc8b7082203c41fbb9d8911dcde3c508 is 1080, key is row0207/info:/1733164395814/Put/seqid=0
2024-12-02T18:33:15,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741873_1049 (size=19013)
2024-12-02T18:33:15,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741873_1049 (size=19013)
2024-12-02T18:33:15,848 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cc8b7082203c41fbb9d8911dcde3c508
2024-12-02T18:33:15,853 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#88 average throughput is 74.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:15,853 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b7cdb3e68e674190b152f28f7f05b409 is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/cc8b7082203c41fbb9d8911dcde3c508 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508
2024-12-02T18:33:15,859 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508, entries=13, sequenceid=292, filesize=18.6 K
2024-12-02T18:33:15,860 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=5.25 KB/5380 for f517d8dad759fdcbfc38dbdc1488f610 in 22ms, sequenceid=292, compaction requested=false
2024-12-02T18:33:15,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:15,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741874_1050 (size=162458)
2024-12-02T18:33:15,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741874_1050 (size=162458)
2024-12-02T18:33:15,867 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/b7cdb3e68e674190b152f28f7f05b409 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b7cdb3e68e674190b152f28f7f05b409
2024-12-02T18:33:15,872 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into b7cdb3e68e674190b152f28f7f05b409(size=158.7 K), total size for store is 177.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:15,872 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:15,872 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164395836; duration=0sec
2024-12-02T18:33:15,872 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:15,872 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:16,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:16,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:17,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:17,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:17,859 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:33:17,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/61a458fef5c94521a08334eab3ed5dc1 is 1080, key is row0220/info:/1733164395839/Put/seqid=0
2024-12-02T18:33:17,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741875_1051 (size=12523)
2024-12-02T18:33:17,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741875_1051 (size=12523)
2024-12-02T18:33:17,873 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/61a458fef5c94521a08334eab3ed5dc1
2024-12-02T18:33:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/61a458fef5c94521a08334eab3ed5dc1 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1
2024-12-02T18:33:17,888 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1, entries=7, sequenceid=303, filesize=12.2 K
2024-12-02T18:33:17,889 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for f517d8dad759fdcbfc38dbdc1488f610 in 29ms, sequenceid=303, compaction requested=true
2024-12-02T18:33:17,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:17,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:17,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:17,889 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:17,890 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-02T18:33:17,890 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 193994 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:17,890 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:17,890 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:17,890 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b7cdb3e68e674190b152f28f7f05b409, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=189.4 K
2024-12-02T18:33:17,891 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7cdb3e68e674190b152f28f7f05b409, keycount=145, bloomtype=ROW, size=158.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733164375441
2024-12-02T18:33:17,891 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc8b7082203c41fbb9d8911dcde3c508, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733164395814
2024-12-02T18:33:17,891 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a458fef5c94521a08334eab3ed5dc1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733164395839
2024-12-02T18:33:17,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1d65fb64784e4e95ac6c6a4695a92c26 is 1080, key is row0227/info:/1733164397862/Put/seqid=0
2024-12-02T18:33:17,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741876_1052 (size=19013)
2024-12-02T18:33:17,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741876_1052 (size=19013)
2024-12-02T18:33:17,899 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1d65fb64784e4e95ac6c6a4695a92c26
2024-12-02T18:33:17,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/1d65fb64784e4e95ac6c6a4695a92c26 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26
2024-12-02T18:33:17,905 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#91 average throughput is 42.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:17,905 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/59631b9d4b894b4b8d15a5befac4feab is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:17,910 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26, entries=13, sequenceid=319, filesize=18.6 K
2024-12-02T18:33:17,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741877_1053 (size=184144)
2024-12-02T18:33:17,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741877_1053 (size=184144)
2024-12-02T18:33:17,911 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=8.41 KB/8608 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=319, compaction requested=false
2024-12-02T18:33:17,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43681 {}] regionserver.HRegion(8855): Flush requested on f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:17,913 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-02T18:33:17,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/4aabf61fb5ec4a62b76fbf1175c0825c is 1080, key is row0240/info:/1733164397891/Put/seqid=0
2024-12-02T18:33:17,920 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/59631b9d4b894b4b8d15a5befac4feab as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/59631b9d4b894b4b8d15a5befac4feab
2024-12-02T18:33:17,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741878_1054 (size=15760)
2024-12-02T18:33:17,925 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741878_1054 (size=15760)
2024-12-02T18:33:17,925 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/4aabf61fb5ec4a62b76fbf1175c0825c
2024-12-02T18:33:17,927 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 59631b9d4b894b4b8d15a5befac4feab(size=179.8 K), total size for store is 198.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:17,927 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:17,927 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164397889; duration=0sec
2024-12-02T18:33:17,928 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:17,928 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:17,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/4aabf61fb5ec4a62b76fbf1175c0825c as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c
2024-12-02T18:33:17,934 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c, entries=10, sequenceid=332, filesize=15.4 K
2024-12-02T18:33:17,935 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=7.36 KB/7532 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=332, compaction requested=true
2024-12-02T18:33:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f517d8dad759fdcbfc38dbdc1488f610:info, priority=-2147483648, current under compaction store size is 1
2024-12-02T18:33:17,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:17,935 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-02T18:33:17,936 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 218917 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-02T18:33:17,936 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1541): f517d8dad759fdcbfc38dbdc1488f610/info is initiating minor compaction (all files)
2024-12-02T18:33:17,936 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f517d8dad759fdcbfc38dbdc1488f610/info in TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:17,936 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/59631b9d4b894b4b8d15a5befac4feab, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c] into tmpdir=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp, totalSize=213.8 K
2024-12-02T18:33:17,936 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59631b9d4b894b4b8d15a5befac4feab, keycount=165, bloomtype=ROW, size=179.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733164375441
2024-12-02T18:33:17,936 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d65fb64784e4e95ac6c6a4695a92c26, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733164397862
2024-12-02T18:33:17,937 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aabf61fb5ec4a62b76fbf1175c0825c, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733164397891
2024-12-02T18:33:17,941 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-12-02T18:33:17,947 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f517d8dad759fdcbfc38dbdc1488f610#info#compaction#93 average throughput is 64.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-02T18:33:17,947 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/7449b7cd71a24e29a2db4017597daf74 is 1080, key is row0062/info:/1733164375441/Put/seqid=0
2024-12-02T18:33:17,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741879_1055 (size=209140)
2024-12-02T18:33:17,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741879_1055 (size=209140)
2024-12-02T18:33:17,955 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/7449b7cd71a24e29a2db4017597daf74 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/7449b7cd71a24e29a2db4017597daf74
2024-12-02T18:33:17,960 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f517d8dad759fdcbfc38dbdc1488f610/info of f517d8dad759fdcbfc38dbdc1488f610 into 7449b7cd71a24e29a2db4017597daf74(size=204.2 K), total size for store is 204.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-02T18:33:17,960 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:17,960 INFO  [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., storeName=f517d8dad759fdcbfc38dbdc1488f610/info, priority=13, startTime=1733164397935; duration=0sec
2024-12-02T18:33:17,960 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-02T18:33:17,960 DEBUG [RS:0;95451c90bed2:43681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f517d8dad759fdcbfc38dbdc1488f610:info
2024-12-02T18:33:18,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:18,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:19,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:19,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:19,928 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-02T18:33:19,929 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43681%2C1733164351982.1733164399929
2024-12-02T18:33:19,943 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,943 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,943 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,943 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,943 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,943 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164352606 with entries=319, filesize=310.54 KB; new WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164399929
2024-12-02T18:33:19,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39097:39097),(127.0.0.1/127.0.0.1:35877:35877)]
2024-12-02T18:33:19,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164352606 is not closed yet, will try archiving it next time
2024-12-02T18:33:19,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741833_1009 (size=317998)
2024-12-02T18:33:19,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741833_1009 (size=317998)
2024-12-02T18:33:19,949 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d68881a7558f42336fade15d797e26db:

2024-12-02T18:33:19,949 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing f517d8dad759fdcbfc38dbdc1488f610 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-02T18:33:19,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/0efcfc5632c344ef9166e5240742a559 is 1080, key is row0250/info:/1733164397914/Put/seqid=0
2024-12-02T18:33:19,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741881_1057 (size=12523)
2024-12-02T18:33:19,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741881_1057 (size=12523)
2024-12-02T18:33:19,960 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/0efcfc5632c344ef9166e5240742a559
2024-12-02T18:33:19,966 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/.tmp/info/0efcfc5632c344ef9166e5240742a559 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/0efcfc5632c344ef9166e5240742a559
2024-12-02T18:33:19,970 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/0efcfc5632c344ef9166e5240742a559, entries=7, sequenceid=344, filesize=12.2 K
2024-12-02T18:33:19,971 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for f517d8dad759fdcbfc38dbdc1488f610 in 21ms, sequenceid=344, compaction requested=false
2024-12-02T18:33:19,971 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f517d8dad759fdcbfc38dbdc1488f610:

2024-12-02T18:33:19,971 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-12-02T18:33:19,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/2c9d94ad6c4a4825a0fc5857889d07a5 is 193, key is TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610./info:regioninfo/1733164376192/Put/seqid=0
2024-12-02T18:33:19,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741882_1058 (size=6223)
2024-12-02T18:33:19,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741882_1058 (size=6223)
2024-12-02T18:33:19,979 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/2c9d94ad6c4a4825a0fc5857889d07a5
2024-12-02T18:33:19,984 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/.tmp/info/2c9d94ad6c4a4825a0fc5857889d07a5 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/info/2c9d94ad6c4a4825a0fc5857889d07a5
2024-12-02T18:33:19,988 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/info/2c9d94ad6c4a4825a0fc5857889d07a5, entries=5, sequenceid=21, filesize=6.1 K
2024-12-02T18:33:19,989 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-12-02T18:33:19,989 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-02T18:33:19,990 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C43681%2C1733164351982.1733164399990
2024-12-02T18:33:19,995 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,995 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,995 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,995 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,995 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:19,995 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164399929 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164399990
2024-12-02T18:33:19,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35877:35877),(127.0.0.1/127.0.0.1:39097:39097)]
2024-12-02T18:33:19,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164399929 is not closed yet, will try archiving it next time
2024-12-02T18:33:19,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741880_1056 (size=731)
2024-12-02T18:33:19,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741880_1056 (size=731)
2024-12-02T18:33:19,997 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164352606 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs/95451c90bed2%2C43681%2C1733164351982.1733164352606
2024-12-02T18:33:19,998 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-02T18:33:19,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:33:19,998 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:33:19,998 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/WALs/95451c90bed2,43681,1733164351982/95451c90bed2%2C43681%2C1733164351982.1733164399929 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs/95451c90bed2%2C43681%2C1733164351982.1733164399929
2024-12-02T18:33:19,998 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:19,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:19,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:19,999 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:33:19,999 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:33:19,999 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1321148526, stopped=false
2024-12-02T18:33:19,999 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,33279,1733164351816
2024-12-02T18:33:20,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:20,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:20,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:20,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:20,047 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:33:20,047 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:33:20,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:20,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:20,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:20,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:20,048 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,43681,1733164351982' *****
2024-12-02T18:33:20,048 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:33:20,048 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(3091): Received CLOSE for d68881a7558f42336fade15d797e26db
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(3091): Received CLOSE for f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,43681,1733164351982
2024-12-02T18:33:20,048 INFO  [RS:0;95451c90bed2:43681 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:33:20,048 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d68881a7558f42336fade15d797e26db, disabling compactions & flushes
2024-12-02T18:33:20,049 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:43681.
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db. after waiting 0 ms
2024-12-02T18:33:20,049 DEBUG [RS:0;95451c90bed2:43681 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:33:20,049 DEBUG [RS:0;95451c90bed2:43681 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:33:20,049 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-02T18:33:20,049 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1325): Online Regions={d68881a7558f42336fade15d797e26db=TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db., f517d8dad759fdcbfc38dbdc1488f610=TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610., 1588230740=hbase:meta,,1.1588230740}
2024-12-02T18:33:20,049 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-bottom] to archive
2024-12-02T18:33:20,049 DEBUG [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d68881a7558f42336fade15d797e26db, f517d8dad759fdcbfc38dbdc1488f610
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:33:20,049 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:33:20,049 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:33:20,050 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:33:20,052 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:33:20,052 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=95451c90bed2:33279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-02T18:33:20,052 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-02T18:33:20,053 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-02T18:33:20,054 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:33:20,054 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:33:20,054 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164400049Running coprocessor pre-close hooks at 1733164400049Disabling compacts and flushes for region at 1733164400049Disabling writes for close at 1733164400049Writing region close event to WAL at 1733164400050 (+1 ms)Running coprocessor post-close hooks at 1733164400054 (+4 ms)Closed at 1733164400054
2024-12-02T18:33:20,054 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:33:20,055 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/d68881a7558f42336fade15d797e26db/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=85
2024-12-02T18:33:20,056 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d68881a7558f42336fade15d797e26db:
Waiting for close lock at 1733164400048Running coprocessor pre-close hooks at 1733164400048Disabling compacts and flushes for region at 1733164400048Disabling writes for close at 1733164400049 (+1 ms)Writing region close event to WAL at 1733164400053 (+4 ms)Running coprocessor post-close hooks at 1733164400056 (+3 ms)Closed at 1733164400056
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733164375512.d68881a7558f42336fade15d797e26db.
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f517d8dad759fdcbfc38dbdc1488f610, disabling compactions & flushes
2024-12-02T18:33:20,056 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610. after waiting 0 ms
2024-12-02T18:33:20,056 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:20,057 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8->hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/ec829d4c71c771c7e68b3e02000f86a8/info/2244dd98676c470b9d4029f23fac2551-top, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b6f98f0ab864465487f2d0b6bcff04cf, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/670fb0397946450782e880b1dc55bf1c, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/62b3da581804430b9d2626e7f0182ecf, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41e181c702124c57b4a856b48d4db0d4, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41252cd8c99d4eea9912b26b00eff291, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a42c73160f7f4746bb43f0653336a42c, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/551040d256c44dfd9591b0179a8cdb52, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b7cdb3e68e674190b152f28f7f05b409, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/59631b9d4b894b4b8d15a5befac4feab, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c] to archive
2024-12-02T18:33:20,058 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-02T18:33:20,059 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/2244dd98676c470b9d4029f23fac2551.ec829d4c71c771c7e68b3e02000f86a8
2024-12-02T18:33:20,060 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b6f98f0ab864465487f2d0b6bcff04cf to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b6f98f0ab864465487f2d0b6bcff04cf
2024-12-02T18:33:20,062 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cb671e08fcc04291b695c9a0c6830ef7
2024-12-02T18:33:20,063 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/670fb0397946450782e880b1dc55bf1c to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/670fb0397946450782e880b1dc55bf1c
2024-12-02T18:33:20,064 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/57d3f34d4d3349f894b8ec2c35e22096
2024-12-02T18:33:20,065 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b16c4bbdee314cbd9fd59f74307e08da
2024-12-02T18:33:20,066 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/62b3da581804430b9d2626e7f0182ecf to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/62b3da581804430b9d2626e7f0182ecf
2024-12-02T18:33:20,067 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/6318078d5c1443b6a823e74513d859a9
2024-12-02T18:33:20,068 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/df013b38a5e44a3abaea5aa24a15b7c6
2024-12-02T18:33:20,068 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41e181c702124c57b4a856b48d4db0d4 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41e181c702124c57b4a856b48d4db0d4
2024-12-02T18:33:20,069 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/c4ff2ad519484eeaa03e011a35d75fcf
2024-12-02T18:33:20,070 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/21eb26b9991c4e7a823672fee6359bc8
2024-12-02T18:33:20,071 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41252cd8c99d4eea9912b26b00eff291 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/41252cd8c99d4eea9912b26b00eff291
2024-12-02T18:33:20,072 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/221f12dca58043ae8b1f1e9fe6862678
2024-12-02T18:33:20,073 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f75624dd53324b7dab8ae4c77cf4e310
2024-12-02T18:33:20,074 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a42c73160f7f4746bb43f0653336a42c to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a42c73160f7f4746bb43f0653336a42c
2024-12-02T18:33:20,075 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1f41f39dbc094a0286d876994c3c1cc5
2024-12-02T18:33:20,076 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b414505b4df046e6bf08c0340b9c36df
2024-12-02T18:33:20,077 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/551040d256c44dfd9591b0179a8cdb52 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/551040d256c44dfd9591b0179a8cdb52
2024-12-02T18:33:20,078 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/32d41b6a944242d08f10751decb5f076
2024-12-02T18:33:20,079 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/f53198a2a6f04578b17906307eba0cdd
2024-12-02T18:33:20,079 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b7cdb3e68e674190b152f28f7f05b409 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/b7cdb3e68e674190b152f28f7f05b409
2024-12-02T18:33:20,080 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/a2067cb3dcd24bfc9d6da415b6b730c5
2024-12-02T18:33:20,081 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/cc8b7082203c41fbb9d8911dcde3c508
2024-12-02T18:33:20,082 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/59631b9d4b894b4b8d15a5befac4feab to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/59631b9d4b894b4b8d15a5befac4feab
2024-12-02T18:33:20,083 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/61a458fef5c94521a08334eab3ed5dc1
2024-12-02T18:33:20,084 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26 to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/1d65fb64784e4e95ac6c6a4695a92c26
2024-12-02T18:33:20,085 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c to hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/archive/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/info/4aabf61fb5ec4a62b76fbf1175c0825c
2024-12-02T18:33:20,085 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b6f98f0ab864465487f2d0b6bcff04cf=8260, cb671e08fcc04291b695c9a0c6830ef7=12509, 670fb0397946450782e880b1dc55bf1c=27778, 57d3f34d4d3349f894b8ec2c35e22096=16817, b16c4bbdee314cbd9fd59f74307e08da=17894, 62b3da581804430b9d2626e7f0182ecf=48385, 6318078d5c1443b6a823e74513d859a9=12515, df013b38a5e44a3abaea5aa24a15b7c6=16828, 41e181c702124c57b4a856b48d4db0d4=71197, c4ff2ad519484eeaa03e011a35d75fcf=15750, 21eb26b9991c4e7a823672fee6359bc8=12516, 41252cd8c99d4eea9912b26b00eff291=91854, 221f12dca58043ae8b1f1e9fe6862678=17906, f75624dd53324b7dab8ae4c77cf4e310=15750, a42c73160f7f4746bb43f0653336a42c=113520, 1f41f39dbc094a0286d876994c3c1cc5=15750, b414505b4df046e6bf08c0340b9c36df=17906, 551040d256c44dfd9591b0179a8cdb52=138621, 32d41b6a944242d08f10751decb5f076=16828, f53198a2a6f04578b17906307eba0cdd=14674, b7cdb3e68e674190b152f28f7f05b409=162458, a2067cb3dcd24bfc9d6da415b6b730c5=19013, cc8b7082203c41fbb9d8911dcde3c508=19013, 59631b9d4b894b4b8d15a5befac4feab=184144, 61a458fef5c94521a08334eab3ed5dc1=12523, 1d65fb64784e4e95ac6c6a4695a92c26=19013, 4aabf61fb5ec4a62b76fbf1175c0825c=15760]
2024-12-02T18:33:20,088 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/data/default/TestLogRolling-testLogRolling/f517d8dad759fdcbfc38dbdc1488f610/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=85
2024-12-02T18:33:20,089 INFO  [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:20,089 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f517d8dad759fdcbfc38dbdc1488f610:
Waiting for close lock at 1733164400056Running coprocessor pre-close hooks at 1733164400056Disabling compacts and flushes for region at 1733164400056Disabling writes for close at 1733164400056Writing region close event to WAL at 1733164400085 (+29 ms)Running coprocessor post-close hooks at 1733164400089 (+4 ms)Closed at 1733164400089
2024-12-02T18:33:20,089 DEBUG [RS_CLOSE_REGION-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733164375512.f517d8dad759fdcbfc38dbdc1488f610.
2024-12-02T18:33:20,249 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,43681,1733164351982; all regions closed.
2024-12-02T18:33:20,250 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,250 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,251 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,251 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,251 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741834_1010 (size=8107)
2024-12-02T18:33:20,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741834_1010 (size=8107)
2024-12-02T18:33:20,261 DEBUG [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs
2024-12-02T18:33:20,261 INFO  [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C43681%2C1733164351982.meta:.meta(num 1733164352934)
2024-12-02T18:33:20,262 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,262 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,262 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,262 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,263 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741883_1059 (size=778)
2024-12-02T18:33:20,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741883_1059 (size=778)
2024-12-02T18:33:20,268 DEBUG [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/oldWALs
2024-12-02T18:33:20,268 INFO  [RS:0;95451c90bed2:43681 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C43681%2C1733164351982:(num 1733164399990)
2024-12-02T18:33:20,268 DEBUG [RS:0;95451c90bed2:43681 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:20,268 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:33:20,268 INFO  [RS:0;95451c90bed2:43681 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:33:20,269 INFO  [RS:0;95451c90bed2:43681 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:33:20,269 INFO  [RS:0;95451c90bed2:43681 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:33:20,269 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:33:20,269 INFO  [RS:0;95451c90bed2:43681 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43681
2024-12-02T18:33:20,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:33:20,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,43681,1733164351982
2024-12-02T18:33:20,279 INFO  [RS:0;95451c90bed2:43681 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:33:20,290 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,43681,1733164351982]
2024-12-02T18:33:20,300 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,43681,1733164351982 already deleted, retry=false
2024-12-02T18:33:20,300 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,43681,1733164351982 expired; onlineServers=0
2024-12-02T18:33:20,300 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,33279,1733164351816' *****
2024-12-02T18:33:20,301 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:33:20,301 INFO  [M:0;95451c90bed2:33279 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:33:20,301 INFO  [M:0;95451c90bed2:33279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:33:20,301 DEBUG [M:0;95451c90bed2:33279 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:33:20,301 DEBUG [M:0;95451c90bed2:33279 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:33:20,301 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:33:20,302 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164352310 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164352310,5,FailOnTimeoutGroup]
2024-12-02T18:33:20,302 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164352310 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164352310,5,FailOnTimeoutGroup]
2024-12-02T18:33:20,302 INFO  [M:0;95451c90bed2:33279 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:33:20,302 INFO  [M:0;95451c90bed2:33279 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:33:20,303 DEBUG [M:0;95451c90bed2:33279 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:33:20,303 INFO  [M:0;95451c90bed2:33279 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:33:20,303 INFO  [M:0;95451c90bed2:33279 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:33:20,303 INFO  [M:0;95451c90bed2:33279 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:33:20,303 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:33:20,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:33:20,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:20,311 DEBUG [M:0;95451c90bed2:33279 {}] zookeeper.ZKUtil(347): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:33:20,311 WARN  [M:0;95451c90bed2:33279 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:33:20,312 INFO  [M:0;95451c90bed2:33279 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/.lastflushedseqids
2024-12-02T18:33:20,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741884_1060 (size=228)
2024-12-02T18:33:20,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741884_1060 (size=228)
2024-12-02T18:33:20,323 INFO  [M:0;95451c90bed2:33279 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:33:20,323 INFO  [M:0;95451c90bed2:33279 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:33:20,323 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:33:20,324 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:20,324 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:20,324 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:33:20,324 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:20,324 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.41 KB heapSize=63.33 KB
2024-12-02T18:33:20,340 DEBUG [M:0;95451c90bed2:33279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3b7eef9213b9427695bbc9a7f996a9ce is 82, key is hbase:meta,,1/info:regioninfo/1733164352956/Put/seqid=0
2024-12-02T18:33:20,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741885_1061 (size=5672)
2024-12-02T18:33:20,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741885_1061 (size=5672)
2024-12-02T18:33:20,345 INFO  [M:0;95451c90bed2:33279 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3b7eef9213b9427695bbc9a7f996a9ce
2024-12-02T18:33:20,362 DEBUG [M:0;95451c90bed2:33279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80225e5bc4624e459c948d44e8d93f1e is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733164353502/Put/seqid=0
2024-12-02T18:33:20,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741886_1062 (size=7089)
2024-12-02T18:33:20,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741886_1062 (size=7089)
2024-12-02T18:33:20,367 INFO  [M:0;95451c90bed2:33279 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.80 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80225e5bc4624e459c948d44e8d93f1e
2024-12-02T18:33:20,371 INFO  [M:0;95451c90bed2:33279 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 80225e5bc4624e459c948d44e8d93f1e
2024-12-02T18:33:20,384 DEBUG [M:0;95451c90bed2:33279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb8b3dfeefac410bb211b364e1fdde1c is 69, key is 95451c90bed2,43681,1733164351982/rs:state/1733164352436/Put/seqid=0
2024-12-02T18:33:20,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741887_1063 (size=5156)
2024-12-02T18:33:20,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741887_1063 (size=5156)
2024-12-02T18:33:20,388 INFO  [M:0;95451c90bed2:33279 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb8b3dfeefac410bb211b364e1fdde1c
2024-12-02T18:33:20,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:20,390 INFO  [RS:0;95451c90bed2:43681 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:33:20,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43681-0x10197513ce10001, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:20,390 INFO  [RS:0;95451c90bed2:43681 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,43681,1733164351982; zookeeper connection closed.
2024-12-02T18:33:20,390 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@561159c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@561159c6
2024-12-02T18:33:20,390 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:33:20,405 DEBUG [M:0;95451c90bed2:33279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/540f7a1d6f4a41f0a2bcf3d02505a033 is 52, key is load_balancer_on/state:d/1733164353122/Put/seqid=0
2024-12-02T18:33:20,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741888_1064 (size=5056)
2024-12-02T18:33:20,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741888_1064 (size=5056)
2024-12-02T18:33:20,410 INFO  [M:0;95451c90bed2:33279 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/540f7a1d6f4a41f0a2bcf3d02505a033
2024-12-02T18:33:20,414 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3b7eef9213b9427695bbc9a7f996a9ce as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3b7eef9213b9427695bbc9a7f996a9ce
2024-12-02T18:33:20,418 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3b7eef9213b9427695bbc9a7f996a9ce, entries=8, sequenceid=125, filesize=5.5 K
2024-12-02T18:33:20,419 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80225e5bc4624e459c948d44e8d93f1e as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80225e5bc4624e459c948d44e8d93f1e
2024-12-02T18:33:20,423 INFO  [M:0;95451c90bed2:33279 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 80225e5bc4624e459c948d44e8d93f1e
2024-12-02T18:33:20,423 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80225e5bc4624e459c948d44e8d93f1e, entries=13, sequenceid=125, filesize=6.9 K
2024-12-02T18:33:20,424 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb8b3dfeefac410bb211b364e1fdde1c as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb8b3dfeefac410bb211b364e1fdde1c
2024-12-02T18:33:20,428 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb8b3dfeefac410bb211b364e1fdde1c, entries=1, sequenceid=125, filesize=5.0 K
2024-12-02T18:33:20,429 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/540f7a1d6f4a41f0a2bcf3d02505a033 as hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/540f7a1d6f4a41f0a2bcf3d02505a033
2024-12-02T18:33:20,433 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44943/user/jenkins/test-data/2fa70ac2-ead3-955a-ca1e-a5bc188e423b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/540f7a1d6f4a41f0a2bcf3d02505a033, entries=1, sequenceid=125, filesize=4.9 K
2024-12-02T18:33:20,434 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=125, compaction requested=false
2024-12-02T18:33:20,435 INFO  [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:20,435 DEBUG [M:0;95451c90bed2:33279 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164400323Disabling compacts and flushes for region at 1733164400323Disabling writes for close at 1733164400324 (+1 ms)Obtaining lock to block concurrent updates at 1733164400324Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164400324Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52639, getHeapSize=64784, getOffHeapSize=0, getCellsCount=148 at 1733164400325 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164400325Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164400325Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164400340 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164400340Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164400349 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164400362 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164400362Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164400371 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164400383 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164400383Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164400392 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164400404 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164400404Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7173f56d: reopening flushed file at 1733164400413 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47ee88c2: reopening flushed file at 1733164400418 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ac14bfe: reopening flushed file at 1733164400423 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d7e562: reopening flushed file at 1733164400428 (+5 ms)Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=125, compaction requested=false at 1733164400434 (+6 ms)Writing region close event to WAL at 1733164400435 (+1 ms)Closed at 1733164400435
2024-12-02T18:33:20,435 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,436 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,436 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,436 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,436 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:20,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37931 is added to blk_1073741830_1006 (size=61308)
2024-12-02T18:33:20,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41107 is added to blk_1073741830_1006 (size=61308)
2024-12-02T18:33:20,438 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:33:20,438 INFO  [M:0;95451c90bed2:33279 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:33:20,438 INFO  [M:0;95451c90bed2:33279 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33279
2024-12-02T18:33:20,438 INFO  [M:0;95451c90bed2:33279 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:33:20,461 INFO  [regionserver/95451c90bed2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:33:20,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:20,548 INFO  [M:0;95451c90bed2:33279 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:33:20,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33279-0x10197513ce10000, quorum=127.0.0.1:51665, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:20,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2998c4ad{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:20,553 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9caeb33{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:20,553 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:20,554 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dc4994c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:20,554 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@216b0c63{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:20,557 WARN  [BP-1257207442-172.17.0.3-1733164349243 heartbeating to localhost/127.0.0.1:44943 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:33:20,557 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:33:20,557 WARN  [BP-1257207442-172.17.0.3-1733164349243 heartbeating to localhost/127.0.0.1:44943 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1257207442-172.17.0.3-1733164349243 (Datanode Uuid 73b8d92b-9718-4fca-bb02-de9a72875ec0) service to localhost/127.0.0.1:44943
2024-12-02T18:33:20,557 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:33:20,558 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data3/current/BP-1257207442-172.17.0.3-1733164349243 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:20,558 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data4/current/BP-1257207442-172.17.0.3-1733164349243 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:20,559 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:33:20,561 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c9ba3de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:20,561 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70d55230{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:20,561 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:20,562 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3523e770{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:20,562 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31e425dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:20,563 WARN  [BP-1257207442-172.17.0.3-1733164349243 heartbeating to localhost/127.0.0.1:44943 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:33:20,563 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:33:20,563 WARN  [BP-1257207442-172.17.0.3-1733164349243 heartbeating to localhost/127.0.0.1:44943 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1257207442-172.17.0.3-1733164349243 (Datanode Uuid c83283a7-65c0-4a22-9875-bad767033ef2) service to localhost/127.0.0.1:44943
2024-12-02T18:33:20,563 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:33:20,564 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data1/current/BP-1257207442-172.17.0.3-1733164349243 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:20,564 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/cluster_d956e9c2-4584-c4ab-488e-ffa0e005f69b/data/data2/current/BP-1257207442-172.17.0.3-1733164349243 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:20,564 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:33:20,568 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2735da07{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:33:20,569 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@425d5d71{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:20,569 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:20,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1776bdc2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:20,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dab95de{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:20,575 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:33:20,604 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:33:20,613 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 206)
Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44943
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44943
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44943
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44943 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44943 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44943
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44943 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44943
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 302), ProcessCount=11 (was 11), AvailableMemoryMB=4705 (was 3954) - AvailableMemoryMB LEAK? -
2024-12-02T18:33:20,619 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=302, ProcessCount=11, AvailableMemoryMB=4705
2024-12-02T18:33:20,619 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.log.dir so I do NOT create it in target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/782dc09e-fc9a-fc00-a42b-1d0926b5b677/hadoop.tmp.dir so I do NOT create it in target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac, deleteOnExit=true
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/test.cache.data in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.tmp.dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-02T18:33:20,620 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-02T18:33:20,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/nfs.dump.dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/java.io.tmpdir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-02T18:33:20,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-02T18:33:20,632 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:33:20,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:20,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:21,105 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:33:21,108 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:33:21,109 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:33:21,109 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:33:21,109 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:33:21,109 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:33:21,109 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b4a20b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:33:21,110 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d0ad989{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:33:21,199 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2527c84a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/java.io.tmpdir/jetty-localhost-41897-hadoop-hdfs-3_4_1-tests_jar-_-any-18074223992025090791/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:33:21,199 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e56b299{HTTP/1.1, (http/1.1)}{localhost:41897}
2024-12-02T18:33:21,199 INFO  [Time-limited test {}] server.Server(415): Started @296586ms
2024-12-02T18:33:21,209 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-02T18:33:21,537 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:33:21,540 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:33:21,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:33:21,541 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:33:21,541 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:33:21,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@271e0d58{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:33:21,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7216654a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:33:21,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:33:21,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-02T18:33:21,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-02T18:33:21,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-02T18:33:21,633 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f4abee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/java.io.tmpdir/jetty-localhost-38217-hadoop-hdfs-3_4_1-tests_jar-_-any-12477969178692726311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:21,633 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c0b8795{HTTP/1.1, (http/1.1)}{localhost:38217}
2024-12-02T18:33:21,633 INFO  [Time-limited test {}] server.Server(415): Started @297019ms
2024-12-02T18:33:21,634 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:33:21,658 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-02T18:33:21,660 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-02T18:33:21,661 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-02T18:33:21,661 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-02T18:33:21,661 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-02T18:33:21,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c9b811e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,AVAILABLE}
2024-12-02T18:33:21,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1187e2f5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-02T18:33:21,750 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68f30489{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/java.io.tmpdir/jetty-localhost-36459-hadoop-hdfs-3_4_1-tests_jar-_-any-6789370360707120589/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:21,751 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4baed4c4{HTTP/1.1, (http/1.1)}{localhost:36459}
2024-12-02T18:33:21,751 INFO  [Time-limited test {}] server.Server(415): Started @297137ms
2024-12-02T18:33:21,752 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-02T18:33:21,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:21,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:22,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:22,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:23,016 WARN  [Thread-2495 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data1/current/BP-2012640905-172.17.0.3-1733164400635/current, will proceed with Du for space computation calculation, 
2024-12-02T18:33:23,016 WARN  [Thread-2496 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data2/current/BP-2012640905-172.17.0.3-1733164400635/current, will proceed with Du for space computation calculation, 
2024-12-02T18:33:23,036 WARN  [Thread-2459 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:33:23,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd194fd45e461f5d8 with lease ID 0x1f5a325c9a7bc3d6: Processing first storage report for DS-e7960599-df2c-4a29-807f-eb56a887cf9a from datanode DatanodeRegistration(127.0.0.1:41511, datanodeUuid=8caa4a27-3697-438b-bcb7-223853983446, infoPort=43689, infoSecurePort=0, ipcPort=38659, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635)
2024-12-02T18:33:23,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd194fd45e461f5d8 with lease ID 0x1f5a325c9a7bc3d6: from storage DS-e7960599-df2c-4a29-807f-eb56a887cf9a node DatanodeRegistration(127.0.0.1:41511, datanodeUuid=8caa4a27-3697-438b-bcb7-223853983446, infoPort=43689, infoSecurePort=0, ipcPort=38659, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:33:23,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd194fd45e461f5d8 with lease ID 0x1f5a325c9a7bc3d6: Processing first storage report for DS-26100edd-aa43-4027-94ac-83293cc9ed6c from datanode DatanodeRegistration(127.0.0.1:41511, datanodeUuid=8caa4a27-3697-438b-bcb7-223853983446, infoPort=43689, infoSecurePort=0, ipcPort=38659, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635)
2024-12-02T18:33:23,038 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd194fd45e461f5d8 with lease ID 0x1f5a325c9a7bc3d6: from storage DS-26100edd-aa43-4027-94ac-83293cc9ed6c node DatanodeRegistration(127.0.0.1:41511, datanodeUuid=8caa4a27-3697-438b-bcb7-223853983446, infoPort=43689, infoSecurePort=0, ipcPort=38659, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:33:23,144 WARN  [Thread-2506 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data3/current/BP-2012640905-172.17.0.3-1733164400635/current, will proceed with Du for space computation calculation, 
2024-12-02T18:33:23,144 WARN  [Thread-2507 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data4/current/BP-2012640905-172.17.0.3-1733164400635/current, will proceed with Du for space computation calculation, 
2024-12-02T18:33:23,160 WARN  [Thread-2482 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-02T18:33:23,162 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9662ae688359573 with lease ID 0x1f5a325c9a7bc3d7: Processing first storage report for DS-84775d4a-c2f9-4655-9f53-03f2668c0405 from datanode DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f5db2dc1-f82b-46c6-acb6-7400a7f3d361, infoPort=33531, infoSecurePort=0, ipcPort=39861, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635)
2024-12-02T18:33:23,162 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9662ae688359573 with lease ID 0x1f5a325c9a7bc3d7: from storage DS-84775d4a-c2f9-4655-9f53-03f2668c0405 node DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f5db2dc1-f82b-46c6-acb6-7400a7f3d361, infoPort=33531, infoSecurePort=0, ipcPort=39861, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-02T18:33:23,162 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc9662ae688359573 with lease ID 0x1f5a325c9a7bc3d7: Processing first storage report for DS-c82e672c-28f5-48a3-b0cb-273935b1f1d0 from datanode DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f5db2dc1-f82b-46c6-acb6-7400a7f3d361, infoPort=33531, infoSecurePort=0, ipcPort=39861, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635)
2024-12-02T18:33:23,162 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc9662ae688359573 with lease ID 0x1f5a325c9a7bc3d7: from storage DS-c82e672c-28f5-48a3-b0cb-273935b1f1d0 node DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f5db2dc1-f82b-46c6-acb6-7400a7f3d361, infoPort=33531, infoSecurePort=0, ipcPort=39861, storageInfo=lv=-57;cid=testClusterID;nsid=507927016;c=1733164400635), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-02T18:33:23,183 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239
2024-12-02T18:33:23,185 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/zookeeper_0, clientPort=53342, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-02T18:33:23,186 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53342
2024-12-02T18:33:23,186 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,187 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:33:23,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741825_1001 (size=7)
2024-12-02T18:33:23,197 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc with version=8
2024-12-02T18:33:23,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36351/user/jenkins/test-data/9f4d0425-f5cb-cf49-147f-e2f1e7fccd25/hbase-staging
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:33:23,199 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-02T18:33:23,200 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:33:23,200 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39209
2024-12-02T18:33:23,201 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39209 connecting to ZooKeeper ensemble=127.0.0.1:53342
2024-12-02T18:33:23,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:392090x0, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:33:23,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39209-0x1019752059c0000 connected
2024-12-02T18:33:23,343 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,345 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:23,350 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc, hbase.cluster.distributed=false
2024-12-02T18:33:23,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:33:23,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39209
2024-12-02T18:33:23,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39209
2024-12-02T18:33:23,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39209
2024-12-02T18:33:23,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39209
2024-12-02T18:33:23,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39209
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/95451c90bed2:0 server-side Connection retries=45
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-02T18:33:23,368 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-02T18:33:23,369 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34815
2024-12-02T18:33:23,370 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34815 connecting to ZooKeeper ensemble=127.0.0.1:53342
2024-12-02T18:33:23,370 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,372 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348150x0, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-02T18:33:23,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348150x0, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:23,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34815-0x1019752059c0001 connected
2024-12-02T18:33:23,384 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-02T18:33:23,385 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-02T18:33:23,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-02T18:33:23,386 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-02T18:33:23,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34815
2024-12-02T18:33:23,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34815
2024-12-02T18:33:23,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34815
2024-12-02T18:33:23,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34815
2024-12-02T18:33:23,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34815
2024-12-02T18:33:23,401 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;95451c90bed2:39209
2024-12-02T18:33:23,401 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:33:23,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:33:23,405 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-02T18:33:23,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,416 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-02T18:33:23,416 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/95451c90bed2,39209,1733164403199 from backup master directory
2024-12-02T18:33:23,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:33:23,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-02T18:33:23,426 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:33:23,426 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,429 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/hbase.id] with ID: 48b982e8-12ef-4a42-a9a4-e7a19202277f
2024-12-02T18:33:23,430 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/.tmp/hbase.id
2024-12-02T18:33:23,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:33:23,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741826_1002 (size=42)
2024-12-02T18:33:23,435 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/.tmp/hbase.id]:[hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/hbase.id]
2024-12-02T18:33:23,445 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:23,445 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-02T18:33:23,446 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-02T18:33:23,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:33:23,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741827_1003 (size=196)
2024-12-02T18:33:23,465 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-02T18:33:23,466 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-02T18:33:23,466 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:33:23,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:33:23,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741828_1004 (size=1189)
2024-12-02T18:33:23,475 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store
2024-12-02T18:33:23,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:33:23,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741829_1005 (size=34)
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:33:23,483 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:23,483 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:23,483 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164403483Disabling compacts and flushes for region at 1733164403483Disabling writes for close at 1733164403483Writing region close event to WAL at 1733164403483Closed at 1733164403483
2024-12-02T18:33:23,484 WARN  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/.initializing
2024-12-02T18:33:23,484 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/WALs/95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,488 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C39209%2C1733164403199, suffix=, logDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/WALs/95451c90bed2,39209,1733164403199, archiveDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/oldWALs, maxLogs=10
2024-12-02T18:33:23,488 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C39209%2C1733164403199.1733164403488
2024-12-02T18:33:23,493 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/WALs/95451c90bed2,39209,1733164403199/95451c90bed2%2C39209%2C1733164403199.1733164403488
2024-12-02T18:33:23,494 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33531:33531),(127.0.0.1/127.0.0.1:43689:43689)]
2024-12-02T18:33:23,495 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:33:23,495 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:33:23,495 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,495 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,497 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:33:23,499 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-02T18:33:23,499 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,499 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:23,499 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:33:23,500 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-02T18:33:23,500 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,501 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:33:23,501 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:33:23,502 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-02T18:33:23,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:33:23,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-02T18:33:23,504 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-02T18:33:23,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-02T18:33:23,505 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,505 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,506 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,506 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,506 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,507 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-02T18:33:23,507 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-02T18:33:23,509 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:33:23,509 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709785, jitterRate=-0.097461998462677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-02T18:33:23,510 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733164403495Initializing all the Stores at 1733164403496 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164403496Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164403497 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164403497Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164403497Cleaning up temporary data from old regions at 1733164403506 (+9 ms)Region opened successfully at 1733164403510 (+4 ms)
2024-12-02T18:33:23,510 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-02T18:33:23,512 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b71b8d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:33:23,513 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-02T18:33:23,513 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-02T18:33:23,513 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-02T18:33:23,513 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-02T18:33:23,514 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-02T18:33:23,514 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-02T18:33:23,514 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-02T18:33:23,516 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-02T18:33:23,517 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-02T18:33:23,521 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-02T18:33:23,521 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-02T18:33:23,522 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-02T18:33:23,531 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-02T18:33:23,532 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-02T18:33:23,533 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-02T18:33:23,542 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-02T18:33:23,543 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-02T18:33:23,552 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-02T18:33:23,556 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-02T18:33:23,563 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-02T18:33:23,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:23,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:23,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,575 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=95451c90bed2,39209,1733164403199, sessionid=0x1019752059c0000, setting cluster-up flag (Was=false)
2024-12-02T18:33:23,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,626 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-02T18:33:23,629 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:23,679 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-02T18:33:23,680 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=95451c90bed2,39209,1733164403199
2024-12-02T18:33:23,682 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-02T18:33:23,683 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-02T18:33:23,684 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-02T18:33:23,684 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-02T18:33:23,684 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 95451c90bed2,39209,1733164403199
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-02T18:33:23,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:33:23,686 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/95451c90bed2:0, corePoolSize=5, maxPoolSize=5
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/95451c90bed2:0, corePoolSize=10, maxPoolSize=10
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:33:23,687 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,689 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:33:23,689 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-02T18:33:23,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733164433690
2024-12-02T18:33:23,690 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(746): ClusterId : 48b982e8-12ef-4a42-a9a4-e7a19202277f
2024-12-02T18:33:23,690 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-02T18:33:23,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-02T18:33:23,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-02T18:33:23,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-02T18:33:23,690 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-02T18:33:23,691 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-02T18:33:23,691 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-02T18:33:23,691 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,691 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,691 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-02T18:33:23,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-02T18:33:23,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-02T18:33:23,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-02T18:33:23,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-02T18:33:23,692 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-02T18:33:23,692 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164403692,5,FailOnTimeoutGroup]
2024-12-02T18:33:23,693 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164403692,5,FailOnTimeoutGroup]
2024-12-02T18:33:23,693 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,693 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-02T18:33:23,693 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,693 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,701 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-02T18:33:23,701 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-02T18:33:23,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:33:23,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741831_1007 (size=1321)
2024-12-02T18:33:23,702 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-02T18:33:23,703 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc
2024-12-02T18:33:23,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:33:23,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741832_1008 (size=32)
2024-12-02T18:33:23,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:33:23,710 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:33:23,711 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-02T18:33:23,711 DEBUG [RS:0;95451c90bed2:34815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6290443f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=95451c90bed2/172.17.0.3:0
2024-12-02T18:33:23,712 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:33:23,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,712 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:23,713 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:33:23,714 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:33:23,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,714 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:23,715 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:33:23,716 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:33:23,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,716 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:23,716 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:33:23,717 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:33:23,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:23,717 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:23,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:33:23,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740
2024-12-02T18:33:23,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740
2024-12-02T18:33:23,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:33:23,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:33:23,720 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:33:23,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:33:23,722 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-02T18:33:23,723 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=744883, jitterRate=-0.05283229053020477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:33:23,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733164403709Initializing all the Stores at 1733164403710 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164403710Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164403710Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164403710Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164403710Cleaning up temporary data from old regions at 1733164403719 (+9 ms)Region opened successfully at 1733164403723 (+4 ms)
2024-12-02T18:33:23,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:33:23,723 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:33:23,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:33:23,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:33:23,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:33:23,724 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:33:23,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164403723Disabling compacts and flushes for region at 1733164403723Disabling writes for close at 1733164403724 (+1 ms)Writing region close event to WAL at 1733164403724Closed at 1733164403724
2024-12-02T18:33:23,725 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:33:23,725 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-02T18:33:23,725 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-02T18:33:23,725 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;95451c90bed2:34815
2024-12-02T18:33:23,726 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-02T18:33:23,726 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-02T18:33:23,726 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-02T18:33:23,726 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:33:23,726 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(2659): reportForDuty to master=95451c90bed2,39209,1733164403199 with port=34815, startcode=1733164403368
2024-12-02T18:33:23,726 DEBUG [RS:0;95451c90bed2:34815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-02T18:33:23,727 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-02T18:33:23,728 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39329, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-02T18:33:23,729 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39209 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,729 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39209 {}] master.ServerManager(517): Registering regionserver=95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,730 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc
2024-12-02T18:33:23,730 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44253
2024-12-02T18:33:23,730 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-02T18:33:23,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:33:23,742 DEBUG [RS:0;95451c90bed2:34815 {}] zookeeper.ZKUtil(111): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,742 WARN  [RS:0;95451c90bed2:34815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-02T18:33:23,742 INFO  [RS:0;95451c90bed2:34815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:33:23,742 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,742 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [95451c90bed2,34815,1733164403368]
2024-12-02T18:33:23,745 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-02T18:33:23,746 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-02T18:33:23,747 INFO  [RS:0;95451c90bed2:34815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-02T18:33:23,747 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,747 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-02T18:33:23,748 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-02T18:33:23,748 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/95451c90bed2:0, corePoolSize=2, maxPoolSize=2
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/95451c90bed2:0, corePoolSize=1, maxPoolSize=1
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:33:23,748 DEBUG [RS:0;95451c90bed2:34815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/95451c90bed2:0, corePoolSize=3, maxPoolSize=3
2024-12-02T18:33:23,748 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,749 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,749 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,749 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,749 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,749 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,34815,1733164403368-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:33:23,760 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-02T18:33:23,760 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,34815,1733164403368-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,760 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,760 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.Replication(171): 95451c90bed2,34815,1733164403368 started
2024-12-02T18:33:23,771 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:23,771 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1482): Serving as 95451c90bed2,34815,1733164403368, RpcServer on 95451c90bed2/172.17.0.3:34815, sessionid=0x1019752059c0001
2024-12-02T18:33:23,772 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-02T18:33:23,772 DEBUG [RS:0;95451c90bed2:34815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,772 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,34815,1733164403368'
2024-12-02T18:33:23,772 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-02T18:33:23,772 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 95451c90bed2,34815,1733164403368
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '95451c90bed2,34815,1733164403368'
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-02T18:33:23,773 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-02T18:33:23,774 DEBUG [RS:0;95451c90bed2:34815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-02T18:33:23,774 INFO  [RS:0;95451c90bed2:34815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-02T18:33:23,774 INFO  [RS:0;95451c90bed2:34815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-02T18:33:23,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:23,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:23,877 INFO  [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C34815%2C1733164403368, suffix=, logDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/95451c90bed2,34815,1733164403368, archiveDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs, maxLogs=32
2024-12-02T18:33:23,877 WARN  [95451c90bed2:39209 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-02T18:33:23,878 INFO  [RS:0;95451c90bed2:34815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C34815%2C1733164403368.1733164403878
2024-12-02T18:33:23,887 INFO  [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/95451c90bed2,34815,1733164403368/95451c90bed2%2C34815%2C1733164403368.1733164403878
2024-12-02T18:33:23,888 DEBUG [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33531:33531),(127.0.0.1/127.0.0.1:43689:43689)]
2024-12-02T18:33:24,128 DEBUG [95451c90bed2:39209 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-02T18:33:24,129 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=95451c90bed2,34815,1733164403368
2024-12-02T18:33:24,132 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,34815,1733164403368, state=OPENING
2024-12-02T18:33:24,194 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-02T18:33:24,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:24,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:24,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:33:24,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:33:24,206 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-02T18:33:24,206 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,34815,1733164403368}]
2024-12-02T18:33:24,358 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-02T18:33:24,362 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39935, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-02T18:33:24,366 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-02T18:33:24,366 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:33:24,368 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=95451c90bed2%2C34815%2C1733164403368.meta, suffix=.meta, logDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/95451c90bed2,34815,1733164403368, archiveDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs, maxLogs=32
2024-12-02T18:33:24,368 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 95451c90bed2%2C34815%2C1733164403368.meta.1733164404368.meta
2024-12-02T18:33:24,374 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/95451c90bed2,34815,1733164403368/95451c90bed2%2C34815%2C1733164403368.meta.1733164404368.meta
2024-12-02T18:33:24,378 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33531:33531),(127.0.0.1/127.0.0.1:43689:43689)]
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-02T18:33:24,382 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-02T18:33:24,382 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-02T18:33:24,384 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-02T18:33:24,384 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-02T18:33:24,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:24,385 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:24,385 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-02T18:33:24,386 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-02T18:33:24,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:24,386 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:24,386 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-02T18:33:24,387 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-02T18:33:24,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:24,387 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:24,387 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-02T18:33:24,388 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-02T18:33:24,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-02T18:33:24,388 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-02T18:33:24,388 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-02T18:33:24,389 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740
2024-12-02T18:33:24,390 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740
2024-12-02T18:33:24,391 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-02T18:33:24,391 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-02T18:33:24,391 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-02T18:33:24,392 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-02T18:33:24,393 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854037, jitterRate=0.08596426248550415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-02T18:33:24,393 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-02T18:33:24,393 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733164404383Writing region info on filesystem at 1733164404383Initializing all the Stores at 1733164404383Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164404383Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164404384 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733164404384Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733164404384Cleaning up temporary data from old regions at 1733164404391 (+7 ms)Running coprocessor post-open hooks at 1733164404393 (+2 ms)Region opened successfully at 1733164404393
2024-12-02T18:33:24,394 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733164404358
2024-12-02T18:33:24,396 DEBUG [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-02T18:33:24,396 INFO  [RS_OPEN_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-02T18:33:24,397 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=95451c90bed2,34815,1733164403368
2024-12-02T18:33:24,397 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 95451c90bed2,34815,1733164403368, state=OPEN
2024-12-02T18:33:24,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:33:24,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-02T18:33:24,435 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=95451c90bed2,34815,1733164403368
2024-12-02T18:33:24,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:33:24,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-02T18:33:24,439 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-02T18:33:24,439 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=95451c90bed2,34815,1733164403368 in 230 msec
2024-12-02T18:33:24,442 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-02T18:33:24,442 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 714 msec
2024-12-02T18:33:24,443 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-02T18:33:24,443 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-02T18:33:24,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:33:24,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,34815,1733164403368, seqNum=-1]
2024-12-02T18:33:24,445 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:33:24,446 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36439, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:33:24,452 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 768 msec
2024-12-02T18:33:24,452 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733164404452, completionTime=-1
2024-12-02T18:33:24,453 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-02T18:33:24,453 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733164464455
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733164524455
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-95451c90bed2:39209, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,455 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,456 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,458 DEBUG [master/95451c90bed2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.034sec
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-02T18:33:24,460 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-02T18:33:24,461 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-02T18:33:24,461 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-02T18:33:24,464 DEBUG [master/95451c90bed2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-02T18:33:24,464 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-02T18:33:24,464 INFO  [master/95451c90bed2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=95451c90bed2,39209,1733164403199-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-02T18:33:24,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9361ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:33:24,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 95451c90bed2,39209,-1 for getting cluster id
2024-12-02T18:33:24,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-02T18:33:24,494 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48b982e8-12ef-4a42-a9a4-e7a19202277f'
2024-12-02T18:33:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-02T18:33:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48b982e8-12ef-4a42-a9a4-e7a19202277f"

2024-12-02T18:33:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413598f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:33:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [95451c90bed2,39209,-1]
2024-12-02T18:33:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-02T18:33:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,498 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-02T18:33:24,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14298e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-02T18:33:24,500 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-02T18:33:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=95451c90bed2,34815,1733164403368, seqNum=-1]
2024-12-02T18:33:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-02T18:33:24,502 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-02T18:33:24,503 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=95451c90bed2,39209,1733164403199
2024-12-02T18:33:24,504 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-02T18:33:24,506 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-02T18:33:24,506 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-02T18:33:24,508 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs, maxLogs=32
2024-12-02T18:33:24,508 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733164404508
2024-12-02T18:33:24,512 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733164404508
2024-12-02T18:33:24,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43689:43689),(127.0.0.1/127.0.0.1:33531:33531)]
2024-12-02T18:33:24,514 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733164404514
2024-12-02T18:33:24,518 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,519 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,519 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,519 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,519 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,519 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733164404508 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733164404514
2024-12-02T18:33:24,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43689:43689),(127.0.0.1/127.0.0.1:33531:33531)]
2024-12-02T18:33:24,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733164404508 is not closed yet, will try archiving it next time
2024-12-02T18:33:24,520 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,521 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741835_1011 (size=93)
2024-12-02T18:33:24,521 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,521 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,521 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,522 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733164404508 to hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs/test.com%2C8080%2C1.1733164404508
2024-12-02T18:33:24,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741835_1011 (size=93)
2024-12-02T18:33:24,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741836_1012 (size=93)
2024-12-02T18:33:24,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741836_1012 (size=93)
2024-12-02T18:33:24,525 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs
2024-12-02T18:33:24,525 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733164404514)
2024-12-02T18:33:24,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-02T18:33:24,525 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:33:24,526 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:24,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,526 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-02T18:33:24,526 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-02T18:33:24,526 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2097695870, stopped=false
2024-12-02T18:33:24,526 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=95451c90bed2,39209,1733164403199
2024-12-02T18:33:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-02T18:33:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:24,571 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:33:24,571 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-02T18:33:24,572 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:24,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,572 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:24,572 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '95451c90bed2,34815,1733164403368' *****
2024-12-02T18:33:24,572 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-02T18:33:24,572 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-02T18:33:24,573 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(959): stopping server 95451c90bed2,34815,1733164403368
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:33:24,573 INFO  [RS:0;95451c90bed2:34815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;95451c90bed2:34815.
2024-12-02T18:33:24,574 DEBUG [RS:0;95451c90bed2:34815 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-02T18:33:24,574 DEBUG [RS:0;95451c90bed2:34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,574 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-02T18:33:24,574 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-02T18:33:24,574 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-02T18:33:24,574 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-02T18:33:24,574 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-02T18:33:24,574 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-02T18:33:24,575 DEBUG [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-02T18:33:24,575 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-02T18:33:24,575 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-02T18:33:24,575 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-02T18:33:24,575 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-02T18:33:24,575 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-02T18:33:24,575 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-02T18:33:24,596 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/.tmp/ns/0c352015c86f44c88afc3cf229a6f4d1 is 43, key is default/ns:d/1733164404447/Put/seqid=0
2024-12-02T18:33:24,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741837_1013 (size=5153)
2024-12-02T18:33:24,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741837_1013 (size=5153)
2024-12-02T18:33:24,601 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/.tmp/ns/0c352015c86f44c88afc3cf229a6f4d1
2024-12-02T18:33:24,606 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/.tmp/ns/0c352015c86f44c88afc3cf229a6f4d1 as hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/ns/0c352015c86f44c88afc3cf229a6f4d1
2024-12-02T18:33:24,611 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/ns/0c352015c86f44c88afc3cf229a6f4d1, entries=2, sequenceid=6, filesize=5.0 K
2024-12-02T18:33:24,612 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false
2024-12-02T18:33:24,613 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-02T18:33:24,617 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-02T18:33:24,617 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-02T18:33:24,617 INFO  [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-02T18:33:24,617 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733164404575Running coprocessor pre-close hooks at 1733164404575Disabling compacts and flushes for region at 1733164404575Disabling writes for close at 1733164404575Obtaining lock to block concurrent updates at 1733164404575Preparing flush snapshotting stores in 1588230740 at 1733164404575Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733164404576 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733164404577 (+1 ms)Flushing 1588230740/ns: creating writer at 1733164404577Flushing 1588230740/ns: appending metadata at 1733164404596 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733164404596Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b92b2b5: reopening flushed file at 1733164404606 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false at 1733164404612 (+6 ms)Writing region close event to WAL at 1733164404613 (+1 ms)Running coprocessor post-close hooks at 1733164404617 (+4 ms)Closed at 1733164404617
2024-12-02T18:33:24,617 DEBUG [RS_CLOSE_META-regionserver/95451c90bed2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-02T18:33:24,752 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-02T18:33:24,752 INFO  [regionserver/95451c90bed2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-02T18:33:24,775 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(976): stopping server 95451c90bed2,34815,1733164403368; all regions closed.
2024-12-02T18:33:24,776 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,776 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,777 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,777 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,777 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741834_1010 (size=1152)
2024-12-02T18:33:24,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741834_1010 (size=1152)
2024-12-02T18:33:24,786 DEBUG [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs
2024-12-02T18:33:24,786 INFO  [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C34815%2C1733164403368.meta:.meta(num 1733164404368)
2024-12-02T18:33:24,787 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,787 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,787 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,787 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,787 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:24,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741833_1009 (size=93)
2024-12-02T18:33:24,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741833_1009 (size=93)
2024-12-02T18:33:24,790 DEBUG [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/oldWALs
2024-12-02T18:33:24,790 INFO  [RS:0;95451c90bed2:34815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 95451c90bed2%2C34815%2C1733164403368:(num 1733164403878)
2024-12-02T18:33:24,790 DEBUG [RS:0;95451c90bed2:34815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-02T18:33:24,790 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.LeaseManager(133): Closed leases
2024-12-02T18:33:24,791 INFO  [RS:0;95451c90bed2:34815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:33:24,791 INFO  [RS:0;95451c90bed2:34815 {}] hbase.ChoreService(370): Chore service for: regionserver/95451c90bed2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-02T18:33:24,791 INFO  [RS:0;95451c90bed2:34815 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:33:24,791 INFO  [regionserver/95451c90bed2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:33:24,791 INFO  [RS:0;95451c90bed2:34815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34815
2024-12-02T18:33:24,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-02T18:33:24,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/95451c90bed2,34815,1733164403368
2024-12-02T18:33:24,805 INFO  [RS:0;95451c90bed2:34815 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:33:24,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,37951,1733164211301/95451c90bed2%2C37951%2C1733164211301.1733164211537
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:24,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40269/user/jenkins/test-data/5c273376-dfa7-a572-1371-817ffb98b8df/WALs/95451c90bed2,42043,1733164210037/95451c90bed2%2C42043%2C1733164210037.meta.1733164211081.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-02T18:33:24,890 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [95451c90bed2,34815,1733164403368]
2024-12-02T18:33:24,900 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/95451c90bed2,34815,1733164403368 already deleted, retry=false
2024-12-02T18:33:24,900 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 95451c90bed2,34815,1733164403368 expired; onlineServers=0
2024-12-02T18:33:24,900 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '95451c90bed2,39209,1733164403199' *****
2024-12-02T18:33:24,900 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-02T18:33:24,900 INFO  [M:0;95451c90bed2:39209 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-02T18:33:24,901 INFO  [M:0;95451c90bed2:39209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-02T18:33:24,901 DEBUG [M:0;95451c90bed2:39209 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-02T18:33:24,901 DEBUG [M:0;95451c90bed2:39209 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-02T18:33:24,901 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-02T18:33:24,901 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164403692 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.small.0-1733164403692,5,FailOnTimeoutGroup]
2024-12-02T18:33:24,901 DEBUG [master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164403692 {}] cleaner.HFileCleaner(306): Exit Thread[master/95451c90bed2:0:becomeActiveMaster-HFileCleaner.large.0-1733164403692,5,FailOnTimeoutGroup]
2024-12-02T18:33:24,901 INFO  [M:0;95451c90bed2:39209 {}] hbase.ChoreService(370): Chore service for: master/95451c90bed2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-02T18:33:24,902 INFO  [M:0;95451c90bed2:39209 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-02T18:33:24,902 DEBUG [M:0;95451c90bed2:39209 {}] master.HMaster(1795): Stopping service threads
2024-12-02T18:33:24,902 INFO  [M:0;95451c90bed2:39209 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-02T18:33:24,902 INFO  [M:0;95451c90bed2:39209 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-02T18:33:24,902 INFO  [M:0;95451c90bed2:39209 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-02T18:33:24,903 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-02T18:33:24,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-02T18:33:24,911 DEBUG [M:0;95451c90bed2:39209 {}] zookeeper.ZKUtil(347): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-02T18:33:24,911 WARN  [M:0;95451c90bed2:39209 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-02T18:33:24,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-02T18:33:24,912 INFO  [M:0;95451c90bed2:39209 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/.lastflushedseqids
2024-12-02T18:33:24,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741838_1014 (size=99)
2024-12-02T18:33:24,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741838_1014 (size=99)
2024-12-02T18:33:24,921 INFO  [M:0;95451c90bed2:39209 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-02T18:33:24,922 INFO  [M:0;95451c90bed2:39209 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-02T18:33:24,922 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-02T18:33:24,922 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:24,922 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:24,922 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-02T18:33:24,922 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:24,922 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-02T18:33:24,937 DEBUG [M:0;95451c90bed2:39209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3345cf85cab74c8a893e89ddc89bdeb1 is 82, key is hbase:meta,,1/info:regioninfo/1733164404396/Put/seqid=0
2024-12-02T18:33:24,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741839_1015 (size=5672)
2024-12-02T18:33:24,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741839_1015 (size=5672)
2024-12-02T18:33:24,941 INFO  [M:0;95451c90bed2:39209 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3345cf85cab74c8a893e89ddc89bdeb1
2024-12-02T18:33:24,958 DEBUG [M:0;95451c90bed2:39209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35b180b17ce8410997de57e83bb10186 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733164404452/Put/seqid=0
2024-12-02T18:33:24,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741840_1016 (size=5275)
2024-12-02T18:33:24,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741840_1016 (size=5275)
2024-12-02T18:33:24,963 INFO  [M:0;95451c90bed2:39209 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35b180b17ce8410997de57e83bb10186
2024-12-02T18:33:24,983 DEBUG [M:0;95451c90bed2:39209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f51d6f40c417432c9d681bf9290928e6 is 69, key is 95451c90bed2,34815,1733164403368/rs:state/1733164403729/Put/seqid=0
2024-12-02T18:33:24,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741841_1017 (size=5156)
2024-12-02T18:33:24,987 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741841_1017 (size=5156)
2024-12-02T18:33:24,987 INFO  [M:0;95451c90bed2:39209 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f51d6f40c417432c9d681bf9290928e6
2024-12-02T18:33:24,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:24,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34815-0x1019752059c0001, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:24,990 INFO  [RS:0;95451c90bed2:34815 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:33:24,990 INFO  [RS:0;95451c90bed2:34815 {}] regionserver.HRegionServer(1031): Exiting; stopping=95451c90bed2,34815,1733164403368; zookeeper connection closed.
2024-12-02T18:33:24,990 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@dcbc3e1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@dcbc3e1
2024-12-02T18:33:24,990 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-02T18:33:25,005 DEBUG [M:0;95451c90bed2:39209 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c60d0ef631204872b899b1f31793fb2e is 52, key is load_balancer_on/state:d/1733164404505/Put/seqid=0
2024-12-02T18:33:25,009 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741842_1018 (size=5056)
2024-12-02T18:33:25,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741842_1018 (size=5056)
2024-12-02T18:33:25,010 INFO  [M:0;95451c90bed2:39209 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c60d0ef631204872b899b1f31793fb2e
2024-12-02T18:33:25,015 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3345cf85cab74c8a893e89ddc89bdeb1 as hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3345cf85cab74c8a893e89ddc89bdeb1
2024-12-02T18:33:25,019 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3345cf85cab74c8a893e89ddc89bdeb1, entries=8, sequenceid=29, filesize=5.5 K
2024-12-02T18:33:25,020 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35b180b17ce8410997de57e83bb10186 as hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35b180b17ce8410997de57e83bb10186
2024-12-02T18:33:25,024 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35b180b17ce8410997de57e83bb10186, entries=3, sequenceid=29, filesize=5.2 K
2024-12-02T18:33:25,025 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f51d6f40c417432c9d681bf9290928e6 as hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f51d6f40c417432c9d681bf9290928e6
2024-12-02T18:33:25,029 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f51d6f40c417432c9d681bf9290928e6, entries=1, sequenceid=29, filesize=5.0 K
2024-12-02T18:33:25,030 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c60d0ef631204872b899b1f31793fb2e as hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c60d0ef631204872b899b1f31793fb2e
2024-12-02T18:33:25,034 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44253/user/jenkins/test-data/785f6597-f9e3-fc84-df17-2fb7e5dc6fbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c60d0ef631204872b899b1f31793fb2e, entries=1, sequenceid=29, filesize=4.9 K
2024-12-02T18:33:25,035 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=29, compaction requested=false
2024-12-02T18:33:25,036 INFO  [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-02T18:33:25,036 DEBUG [M:0;95451c90bed2:39209 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733164404922Disabling compacts and flushes for region at 1733164404922Disabling writes for close at 1733164404922Obtaining lock to block concurrent updates at 1733164404922Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733164404922Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733164404922Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733164404923 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733164404923Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733164404936 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733164404936Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733164404945 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733164404958 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733164404958Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733164404967 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733164404982 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733164404982Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733164404991 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733164405005 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733164405005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6958750d: reopening flushed file at 1733164405014 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ee851d3: reopening flushed file at 1733164405019 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cc8f5cc: reopening flushed file at 1733164405025 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c62e6ca: reopening flushed file at 1733164405029 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=29, compaction requested=false at 1733164405035 (+6 ms)Writing region close event to WAL at 1733164405036 (+1 ms)Closed at 1733164405036
2024-12-02T18:33:25,037 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:25,037 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:25,037 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:25,037 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:25,037 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-02T18:33:25,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41511 is added to blk_1073741830_1006 (size=10311)
2024-12-02T18:33:25,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741830_1006 (size=10311)
2024-12-02T18:33:25,039 INFO  [M:0;95451c90bed2:39209 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-02T18:33:25,039 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-02T18:33:25,039 INFO  [M:0;95451c90bed2:39209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39209
2024-12-02T18:33:25,039 INFO  [M:0;95451c90bed2:39209 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-02T18:33:25,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,070 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,071 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,071 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-02T18:33:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:25,147 INFO  [M:0;95451c90bed2:39209 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-02T18:33:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39209-0x1019752059c0000, quorum=127.0.0.1:53342, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-02T18:33:25,152 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68f30489{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:25,153 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4baed4c4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:25,153 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:25,154 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1187e2f5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:25,154 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c9b811e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:25,157 WARN  [BP-2012640905-172.17.0.3-1733164400635 heartbeating to localhost/127.0.0.1:44253 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:33:25,157 WARN  [BP-2012640905-172.17.0.3-1733164400635 heartbeating to localhost/127.0.0.1:44253 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2012640905-172.17.0.3-1733164400635 (Datanode Uuid f5db2dc1-f82b-46c6-acb6-7400a7f3d361) service to localhost/127.0.0.1:44253
2024-12-02T18:33:25,157 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:33:25,157 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:33:25,158 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data3/current/BP-2012640905-172.17.0.3-1733164400635 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:25,158 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data4/current/BP-2012640905-172.17.0.3-1733164400635 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:25,159 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:33:25,160 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f4abee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-02T18:33:25,160 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c0b8795{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:25,161 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:25,161 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7216654a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:25,161 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@271e0d58{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:25,161 WARN  [BP-2012640905-172.17.0.3-1733164400635 heartbeating to localhost/127.0.0.1:44253 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-02T18:33:25,161 WARN  [BP-2012640905-172.17.0.3-1733164400635 heartbeating to localhost/127.0.0.1:44253 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2012640905-172.17.0.3-1733164400635 (Datanode Uuid 8caa4a27-3697-438b-bcb7-223853983446) service to localhost/127.0.0.1:44253
2024-12-02T18:33:25,162 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-02T18:33:25,162 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-02T18:33:25,162 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data1/current/BP-2012640905-172.17.0.3-1733164400635 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:25,162 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/cluster_f7f1b68c-56c4-38b0-bdf9-cae6b85590ac/data/data2/current/BP-2012640905-172.17.0.3-1733164400635 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-02T18:33:25,162 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-02T18:33:25,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2527c84a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-02T18:33:25,167 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e56b299{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-02T18:33:25,167 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-02T18:33:25,167 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d0ad989{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-02T18:33:25,167 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b4a20b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49e01f49-2905-1e35-1e8b-e63c29e7f239/hadoop.log.dir/,STOPPED}
2024-12-02T18:33:25,173 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-02T18:33:25,186 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-02T18:33:25,193 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 230)
Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44253
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44253
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44253
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44253
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44253 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44253
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44253 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44253 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 302), ProcessCount=11 (was 11), AvailableMemoryMB=4694 (was 4705)